wiki_dprnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import collections\n",
    "import random\n",
    "from time import time\n",
    "\n",
    "import data_handler as dh\n",
    "\n",
    "# plot settings\n",
    "% matplotlib inline\n",
    "# print(plt.rcParams.keys())\n",
    "plt.rcParams['figure.figsize'] = (16,9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class RNN_Embed(object):\n",
    "    \"\"\" Encapsulation of the dependency RNN lang model\n",
    "    \n",
    "    Largely inspired by https://github.com/tensorflow/tensorflow/blob/master/tensorflow/models/rnn/ptb/ptb_word_lm.py\n",
    "    \"\"\"\n",
    "    def __init__(self, config):\n",
    "#         self.batch_size = config['batch_size']\n",
    "        self.max_num_steps = config['max_num_steps']\n",
    "        self.word_embed_size = config['word_embed_size']\n",
    "        self.dep_embed_size = config['dep_embed_size']\n",
    "        self.input_size = self.word_embed_size + self.dep_embed_size\n",
    "        self.hidden_size = 2 * self.word_embed_size #config['hidden_size']\n",
    "        self.max_grad_norm = config['max_grad_norm']\n",
    "        self.vocab_size = config['vocab_size']\n",
    "        self.dep_vocab_size = config['dep_vocab_size']\n",
    "        self.name = config['model_name']\n",
    "        self.checkpoint_prefix = config['checkpoint_prefix'] + self.name\n",
    "        self.summary_prefix = config['summary_prefix'] + self.name\n",
    "        \n",
    "        self.initializer = tf.random_uniform_initializer(-1., 1.)\n",
    "        with tf.name_scope(self.name):\n",
    "            with tf.name_scope(\"Forward\"):\n",
    "                self._build_forward_graph()\n",
    "            with tf.name_scope(\"Backward\"):\n",
    "                self._build_train_graph()\n",
    "            with tf.name_scope(\"Nearby\"):\n",
    "                self._build_similarity_graph()\n",
    "        \n",
    "        self._valid_accuracy = tf.Variable(0.0, trainable=False)\n",
    "        self._valid_acc_summary = tf.merge_summary([tf.scalar_summary(\"Valid_accuracy\", self._valid_accuracy)])\n",
    "\n",
    "        self.saver = tf.train.Saver(tf.all_variables())\n",
    "            \n",
    "        self.session = tf.InteractiveSession()\n",
    "        self.session.run(tf.initialize_all_variables())        \n",
    "        self.summary_writer = tf.train.SummaryWriter(self.summary_prefix, self.session.graph_def)\n",
    "        \n",
    "    def save_validation_accuracy(self, new_score):\n",
    "        assign_op = self._valid_accuracy.assign(new_score)\n",
    "        _, summary = self.session.run([assign_op, self._valid_acc_summary])\n",
    "        self.summary_writer.add_summary(summary)\n",
    "        \n",
    "    def _build_forward_graph(self):\n",
    "        # input tensor of zero padded indices to get to max_num_steps\n",
    "        # None allows for variable batch sizes\n",
    "        with tf.name_scope(\"Inputs\"):\n",
    "            self._input_phrases = tf.placeholder(tf.int32, [None, self.max_num_steps, 2]) # [batch_size, w_{1:N}, 2]\n",
    "            self._input_targets = tf.placeholder(tf.int32, [None, 2]) # [batch_size, w_x]\n",
    "            self._input_labels = tf.placeholder(tf.int32, [None, 1]) # [batch_size, from true data?] \\in {0,1}\n",
    "            self._input_lengths = tf.placeholder(tf.int32, [None, 1]) # [batch_size, N] (len of each sequence)\n",
    "            batch_size = tf.shape(self._input_lengths)[0]\n",
    "        \n",
    "        with tf.name_scope(\"Embeddings\"):\n",
    "            self._word_embeddings = tf.get_variable(\"word_embeddings\", \n",
    "                                                    [self.vocab_size, self.word_embed_size],\n",
    "                                                    dtype=tf.float32)\n",
    "            self._dependency_embeddings = tf.get_variable(\"dependency_embeddings\", \n",
    "                                                    [self.dep_vocab_size, self.dep_embed_size],\n",
    "                                                    dtype=tf.float32)\n",
    "            self._left_target_embeddings = tf.get_variable(\"left_target_embeddings\", \n",
    "                                                    [self.vocab_size, self.word_embed_size],\n",
    "                                                    dtype=tf.float32)\n",
    "            self._right_target_embeddings = tf.get_variable(\"right_target_embeddings\", \n",
    "                                                    [self.vocab_size, self.word_embed_size],\n",
    "                                                    dtype=tf.float32)\n",
    "            # TODO: Add POS embeddings\n",
    "            \n",
    "            input_embeds = tf.nn.embedding_lookup(self._word_embeddings, \n",
    "                                                  tf.slice(self._input_phrases, [0,0,0], [-1, -1, 1]))\n",
    "            dep_embeds = tf.nn.embedding_lookup(self._dependency_embeddings,\n",
    "                                                tf.slice(self._input_phrases, [0,0,1], [-1, -1, 1]))\n",
    "            left_target_embeds = tf.nn.embedding_lookup(self._left_target_embeddings, \n",
    "                                                        tf.slice(self._input_targets, [0,0], [-1, 1]))\n",
    "            right_target_embeds = tf.nn.embedding_lookup(self._right_target_embeddings, \n",
    "                                                        tf.slice(self._input_targets, [0,1], [-1, 1]))\n",
    "#             print(tf.slice(self._input_phrases, [0,0,1], [-1, -1, 1]).get_shape(), dep_embeds.get_shape())\n",
    "#             print(left_target_embeds.get_shape(), right_target_embeds.get_shape())\n",
    "            self._target_embeds = tf.squeeze(tf.concat(2, [left_target_embeds, right_target_embeds]), [1])\n",
    "#             print(target_embeds.get_shape())\n",
    "            # TODO: Add dropout to embeddings\n",
    "        \n",
    "        with tf.name_scope(\"RNN\"):\n",
    "            # start off with a basic configuration\n",
    "            self.cell = tf.nn.rnn_cell.GRUCell(self.hidden_size, \n",
    "                                                input_size=self.input_size)\n",
    "            # TODO: Add Dropout wrapper\n",
    "            # TODO: Make it multilevel\n",
    "#             self._initial_state = self.cell.zero_state(batch_size, tf.float32)\n",
    "#             print(self._initial_state.get_shape())\n",
    "            input_words = [ tf.squeeze(input_, [1, 2]) for input_ in tf.split(1, self.max_num_steps, input_embeds)]\n",
    "            input_deps = [ tf.squeeze(input_, [1, 2]) for input_ in tf.split(1, self.max_num_steps, dep_embeds)]\n",
    "            inputs = [ tf.concat(1, [input_word, input_dep]) \n",
    "                      for (input_word, input_dep) in zip(input_words, input_deps)]\n",
    "\n",
    "            _, state = tf.nn.rnn(self.cell, inputs, \n",
    "                                 sequence_length=tf.squeeze(self._input_lengths, [1]),\n",
    "                                 dtype=tf.float32)\n",
    "#                                  initial_state=self._initial_state)\n",
    "            self._final_state = state\n",
    "            \n",
    "        with tf.name_scope(\"Loss\"):\n",
    "            flat_states = tf.reshape(state, [-1])\n",
    "            flat_target_embeds = tf.reshape(target_embeds, [-1])\n",
    "#             assert self.hidden_size == (self.word_embed_size), \"Hidden state must equal concated inputs\" \n",
    "            flat_logits = tf.mul(flat_states, flat_target_embeds)\n",
    "            logits = tf.reduce_sum(tf.reshape(flat_logits, tf.pack([batch_size, -1])), 1)\n",
    "            self._loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits, \n",
    "                                                                    tf.to_float(self._input_labels)))\n",
    "            \n",
    "        with tf.name_scope(\"Summaries\"):\n",
    "            \n",
    "            self._train_cost_summary = tf.merge_summary([tf.scalar_summary(\"Train_NEG_Loss\", self._loss)])\n",
    "            self._valid_cost_summary = tf.merge_summary([tf.scalar_summary(\"Validation_NEG_Loss\", self._loss)])\n",
    "        \n",
    "    def _build_train_graph(self):\n",
    "        with tf.name_scope(\"Trainer\"):\n",
    "            self._global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
    "#             self._lr = tf.Variable(1.0, trainable=False)\n",
    "            self._optimizer = tf.train.AdagradOptimizer(1.0)\n",
    "            \n",
    "            # clip and apply gradients\n",
    "            grads_and_vars = self._optimizer.compute_gradients(self._loss)\n",
    "#             for gv in grads_and_vars:\n",
    "#                 print(gv, gv[1] is self._cost)\n",
    "            clipped_grads_and_vars = [(tf.clip_by_norm(gv[0], self.max_grad_norm), gv[1]) \n",
    "                                      for gv in grads_and_vars if gv[0] is not None] # clip_by_norm doesn't like None\n",
    "            \n",
    "            with tf.name_scope(\"Summaries\"):\n",
    "                grad_summaries = []\n",
    "                for g, v in grads_and_vars:\n",
    "                    if g is not None:\n",
    "                        grad_hist_summary = tf.histogram_summary(\"{}/grad/hist\".format(v.name), g)\n",
    "                        sparsity_summary = tf.scalar_summary(\"{}/grad/sparsity\".format(v.name), tf.nn.zero_fraction(g))\n",
    "                        grad_summaries.append(grad_hist_summary)\n",
    "                        grad_summaries.append(sparsity_summary)\n",
    "                self._grad_summaries = tf.merge_summary(grad_summaries)\n",
    "            self._train_op = self._optimizer.apply_gradients(clipped_grads_and_vars, global_step=self._global_step)\n",
    "            \n",
    "    def _build_similarity_graph(self):\n",
    "        tf.get_variable_scope().reuse_variables()\n",
    "        with tf.name_scope(\"Inputs\"):\n",
    "            # word or phrase we want similarities for\n",
    "#             self._query_word = tf.placeholder(tf.int32, [1], name=\"q_word\")\n",
    "            self._query_phrase = tf.placeholder(tf.int32, [self.max_num_steps, 2], name=\"q_phrase\")\n",
    "            self._query_length = tf.placeholder(tf.int32, [1], name=\"q_len\") # lengths for RNN\n",
    "            # words and phrases to compute similarities over\n",
    "#             self._sim_words = tf.placeholder(tf.int32, [None, 1])\n",
    "            self._sim_phrases = tf.placeholder(tf.int32, [None, self.max_num_steps, 2])\n",
    "            self._sim_lengths = tf.placeholder(tf.int32, [None, 1]) # lengths for RNN\n",
    "            sim_size = tf.shape(self._sim_lengths)[0]\n",
    "        \n",
    "        with tf.name_scope(\"Embeddings\"):\n",
    "            query_phrase_embed = tf.nn.embedding_lookup(self._word_embeddings, \n",
    "                                                  tf.slice(self._query_phrase, [0,0], [-1, 1]))\n",
    "            query_dep_embed = tf.nn.embedding_lookup(self._dependency_embeddings,\n",
    "                                                tf.slice(self._query_phrase, [0,1], [-1, 1]))\n",
    "#             query_word_embed = tf.nn.embedding_lookup(self._word_embeddings, self._query_word)\n",
    "#             query_phrase_embed = tf.nn.embedding_lookup(self._word_embeddings, self._query_phrase)\n",
    "#             sim_word_embed = tf.nn.embedding_lookup(self._word_embeddings, tf.squeeze(self._sim_words, [1]))\n",
    "            sim_phrase_embed = tf.nn.embedding_lookup(self._word_embeddings, \n",
    "                                                  tf.slice(self._sim_phrases, [0, 0, 0], [-1, -1, 1]))\n",
    "            sim_dep_embed = tf.nn.embedding_lookup(self._dependency_embeddings, \n",
    "                                                  tf.slice(self._sim_phrases, [0, 0, 1], [-1, -1, 1]))\n",
    "        \n",
    "        with tf.name_scope(\"RNN\"):\n",
    "            # compute rep of a query phrase\n",
    "            query_phrase = [tf.squeeze(qw, [1]) for qw in tf.split(0, self.max_num_steps, query_phrase_embed)]\n",
    "            query_dep = [tf.squeeze(qd, [1]) for qd in tf.split(0, self.max_num_steps, query_dep_embed)]\n",
    "#             print(query_phrase[0].get_shape(), query_dep[0].get_shape())\n",
    "            query_input = [ tf.concat(1, [qw, qd]) for (qw, qd) in zip(query_phrase, query_dep)]\n",
    "            _, query_phrase_state = tf.nn.rnn(self.cell, query_input, \n",
    "                                              sequence_length=self._query_length, \n",
    "                                              dtype=tf.float32)\n",
    "            # compute reps of similarity phrases\n",
    "            sim_phrases = [tf.squeeze(qw, [1,2]) for qw in tf.split(1, self.max_num_steps, sim_phrase_embed)]\n",
    "            sim_deps = [tf.squeeze(qd, [1,2]) for qd in tf.split(1, self.max_num_steps, sim_dep_embed)]\n",
    "            sim_input = [ tf.concat(1, [qw, qd]) for (qw, qd) in zip(sim_phrases, sim_deps)]\n",
    "            _, sim_phrase_states = tf.nn.rnn(self.cell, sim_input, \n",
    "                                             sequence_length=tf.squeeze(self._sim_lengths, [1]), \n",
    "                                             dtype=tf.float32)\n",
    "#             print(query_phrase_state.get_shape(), sim_phrase_states.get_shape())\n",
    "            \n",
    "        with tf.name_scope(\"Similarities\"):\n",
    "            with tf.name_scope(\"Normalize\"):\n",
    "#                 print(query_phrase.get_shape())\n",
    "                query_phrase = tf.nn.l2_normalize(query_phrase_state, 1)\n",
    "#                 query_word = tf.nn.l2_normalize(query_word_embed, 1)\n",
    "                sim_phrases = tf.nn.l2_normalize(sim_phrase_states, 1)\n",
    "#                 sim_word = tf.nn.l2_normalize(sim_word_embed, 1)                \n",
    "#                 print(sim_word.get_shape(), sim_phrases.get_shape())\n",
    "\n",
    "            with tf.name_scope(\"Calc_distances\"):\n",
    "                # do for words\n",
    "#                 print(q)\n",
    "#                 query_word_nearby_dist = tf.matmul(query_word, sim_word, transpose_b=True)\n",
    "#                 qw_nearby_val, qw_nearby_idx = tf.nn.top_k(query_word_nearby_dist, min(1000, self.vocab_size))\n",
    "#                 self.qw_nearby_val = tf.squeeze(qw_nearby_val)\n",
    "#                 self.qw_nearby_idx = tf.squeeze(qw_nearby_idx)\n",
    "#                 self.qw_nearby_words = tf.squeeze(tf.gather(self._sim_words, qw_nearby_idx))\n",
    "\n",
    "                # do for phrases\n",
    "                query_phrase_nearby_dist = tf.matmul(query_phrase, sim_phrases, transpose_b=True)\n",
    "                qp_nearby_val, qp_nearby_idx = tf.nn.top_k(query_phrase_nearby_dist, min(1000, sim_size))\n",
    "                self.qp_nearby_val = tf.squeeze(qp_nearby_val)\n",
    "                self.qp_nearby_idx = tf.squeeze(qp_nearby_idx)\n",
    "#                 self.qp_nearby_lens = tf.squeeze(tf.gather(self._sim_lengths, qp_nearby_idx))\n",
    "            \n",
    "    def partial_fit(self, input_phrases, input_targets, input_labels, input_lengths):\n",
    "        \"\"\"Fit a mini-batch\n",
    "        \n",
    "        Expects a batch_x: [self.batch_size, self.max_num_steps]\n",
    "                  batch_y: the same\n",
    "                  batch_seq_lens: [self.batch_size]\n",
    "                  \n",
    "        Returns average batch perplexity\n",
    "        \"\"\"\n",
    "        loss, _, g_summaries, c_summary = self.session.run([self._loss, self._train_op, \n",
    "                                                            self._grad_summaries,\n",
    "                                                            self._train_cost_summary],\n",
    "                                                           {self._input_phrases:input_phrases,\n",
    "                                                            self._input_targets:input_targets,\n",
    "                                                            self._input_labels:input_labels,\n",
    "                                                            self._input_lengths:input_lengths})\n",
    "        self.summary_writer.add_summary(g_summaries)\n",
    "        self.summary_writer.add_summary(c_summary)\n",
    "        return loss\n",
    "    \n",
    "    def validation_phrase_nearby(self, q_phrase, sim_phrases, sim_phrase_lens):\n",
    "        \"\"\"Return nearby phrases from the similarity set\n",
    "        \"\"\"\n",
    "        query_len = np.array((len(q_phrase),)).astype(np.int32)\n",
    "        nearby_vals, nearby_idx, = self.session.run([self.qp_nearby_val, \n",
    "                                                     self.qp_nearby_idx,],\n",
    "                                                                   {self._query_phrase:q_phrase, \n",
    "                                                                    self._query_length:query_len,\n",
    "                                                                    self._sim_phrases:sim_phrases,\n",
    "                                                                    self._sim_lengths:sim_phrase_lens})\n",
    "        return nearby_vals, nearby_idx\n",
    "    \n",
    "    def encode_phrases_and_targets(self, phrases, targets, lens):\n",
    "        phrases_reps, target_reps = self.session.run([self._final_state, self._target_embeds],\n",
    "                                                    { self._input_phrases:phrases,\n",
    "                                                      self._input_targets:targets,\n",
    "                                                      self._input_lens:lens})\n",
    "\n",
    "    \n",
    "#     def validation_word_nearby(self, q_word, sim_words):\n",
    "#         \"\"\"Return nearby phrases from the similarity set\n",
    "#         \"\"\"\n",
    "#         nearby_vals, nearby_idx = self.session.run([self.qw_nearby_val, \n",
    "#                                                       self.qw_nearby_idx],\n",
    "#                                                        {self._query_word:q_word, \n",
    "#                                                         self._sim_words:sim_words})\n",
    "#         return nearby_vals, nearby_idx\n",
    "        \n",
    "    def predict(self, sequences, seq_lens, return_probs=False):\n",
    "        if return_probs:\n",
    "            predictions, distributions = self.session.run([self._predictions, self._predicted_dists],\n",
    "                                                          {self._predict_inputs:sequences,\n",
    "                                                           self._predict_lengths:seq_lens})\n",
    "            distributions = distributions.reshape([sequences.shape[0], sequences.shape[1], -1])\n",
    "            pred_list = []\n",
    "            dist_list = []\n",
    "            for i, seq_len in enumerate(seq_lens):\n",
    "                pred_list.append(list(predictions[i, :seq_len]))\n",
    "                dist_list.append([distributions[i,j,:] for j in range(seq_len)])\n",
    "            return pred_list, dist_list\n",
    "        \n",
    "        else:\n",
    "            predictions = self.session.run(self._predictions,\n",
    "                                           {self._predict_inputs:sequences,\n",
    "                                            self._predict_lengths:seq_lens})\n",
    "            pred_list = []\n",
    "            for i, seq_len in enumerate(seq_lens):\n",
    "                pred_list.append(list(predictions[i, :seq_len])) \n",
    "            return pred_list\n",
    "            \n",
    "    def checkpoint(self):\n",
    "        self.saver.save(self.session, self.checkpoint_prefix + '.ckpt', global_step=self._global_step)\n",
    "        \n",
    "    def restore(self, model_ckpt_path):\n",
    "        self.saver.restore(self.session, model_ckpt_path)\n",
    "        \n",
    "    def __repr__(self):\n",
    "        return (\"<DPNN: W:%i, D:%i, H:%i, V:%i>\" \n",
    "                % (self.word_embed_size, self.dep_embed_size, self.hidden_size, self.vocab_size))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Data objects...\n",
      "Done creating Data objects\n",
      "763699 total examples :: 756062 training : 7637 valid (99:1 split)\n",
      "Vocab size: 16955 Dep size: 46\n"
     ]
    }
   ],
   "source": [
    "reload(dh)\n",
    "DH = dh.DataHandler('data/wiki_sdp_50000', valid_percent=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33\n"
     ]
    }
   ],
   "source": [
    "print(DH.max_seq_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<DPNN: W:200, D:25, H:400, V:16955>\n"
     ]
    }
   ],
   "source": [
    "# tf.get_default_session().close()\n",
    "# tf.reset_default_graph()\n",
    "k = 20\n",
    "config = {\n",
    "#     'batch_size':100,\n",
    "    'max_num_steps':DH.max_seq_len,\n",
    "    'word_embed_size':200,\n",
    "    'dep_embed_size':25,\n",
    "#     'hidden_size':200,\n",
    "    'vocab_size':DH.vocab_size,\n",
    "    'dep_vocab_size':DH.dep_size,\n",
    "    'max_grad_norm':3.,\n",
    "#     'nce_num_samples':25,\n",
    "    'model_name':'drnn_embed_wiki1',\n",
    "    'checkpoint_prefix':'checkpoints/',\n",
    "    'summary_prefix':'tensor_summaries/'\n",
    "}\n",
    "try:\n",
    "    tf.reset_default_graph()\n",
    "except:\n",
    "    pass\n",
    "try:\n",
    "    tf.get_default_session().close()\n",
    "except:\n",
    "    pass\n",
    "drnn = RNN_Embed(config)\n",
    "print(drnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def run_validation_test(drnn, DH, num_nearby=20):\n",
    "    valid_phrases, valid_lens = DH.validation_batch()\n",
    "    random_index = int(random.uniform(0, len(valid_lens)))\n",
    "    random_index = 0\n",
    "    query_phrase = valid_phrases[random_index]\n",
    "    query_len = valid_lens[random_index]\n",
    "    padded_qp = np.zeros([DH.max_seq_len, 2]).astype(np.int32)\n",
    "    padded_qp[:len(query_phrase), 0] = [x[0] for x in query_phrase]\n",
    "    padded_qp[:len(query_phrase), 1] = [x[1] for x in query_phrase]    \n",
    "    dists, phrase_idx = drnn.validation_phrase_nearby(padded_qp, valid_phrases, valid_lens)\n",
    "    print(\"=\"*80)\n",
    "    print(\"Top %i closest phrases to '%s'\" % (num_nearby, DH.sequence_to_sentence(query_phrase, query_len)))\n",
    "    for i in range(num_nearby):\n",
    "        dist = dists[i]\n",
    "        phrase = valid_phrases[phrase_idx[i]]\n",
    "        len_ = valid_lens[phrase_idx[i]]\n",
    "        print(\"%i: %0.3f : '%s'\" % (i, dist,DH.sequence_to_sentence(phrase, len_)))\n",
    "    print(\"=\"*80)\n",
    "#     drnn.save_validation_accuracy(frac_correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def time_left(num_epochs, num_steps, fit_time, nearby_time, start_time, nearby_mod):\n",
    "    fit = num_steps*fit_time\n",
    "    nearby = (num_steps / float(nearby_mod)) * nearby_time\n",
    "    total = num_epochs*(fit + nearby)\n",
    "    return total - (time() - start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0:0:4) step 0/15121, epoch 0 Training Loss = 0.16423 :: 950.030 phrases/sec :: (5:44:46) hours left\n",
      "================================================================================\n",
      "Top 20 closest phrases to 'University'\n",
      "0: 0.997 : 'of most spent in research meetings in countries including France Switzerland Romania Turkey Britain Belgium Germany Denmark Sweden Greece'\n",
      "1: 0.997 : '<OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> McCormack <OOV> <OOV> Moore <OOV> <OOV> <OOV> <OOV> <OOV> <OOV>'\n",
      "2: 0.997 : '<OOV> <OOV> Todd <OOV> <OOV> <OOV> <OOV> <OOV> Music King <OOV> <OOV> <OOV> <OOV> Gibbons Black <OOV> Orchestra'\n",
      "3: 0.997 : 'model model model model fidelity model fidelity model model model model model model mapped model extraction space space mapping'\n",
      "4: 0.997 : '<OOV> <OOV> <OOV> <OOV> Smith <OOV> Clement <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> Powell <OOV>'\n",
      "5: 0.996 : 'of program with accordance in development of plans statements of <OOV> Party decisions of Council of Ministers of'\n",
      "6: 0.996 : 'found in Spain Italy France <OOV> Germany Austria Switzerland Poland Republic Slovakia Hungary Romania Slovenia Croatia'\n",
      "7: 0.996 : 'India Malaysia Iran Armenia Georgia Turkey Kazakhstan Russia Spain Kingdom Germany France Italy Austria Norway Poland'\n",
      "8: 0.995 : 'in recorded detection in <OOV> of part as used asked series of questions pertaining to issue under'\n",
      "9: 0.995 : 'around parties in workers of organization Department under emerged given <OOV> relative to activities of parties affiliated with'\n",
      "10: 0.995 : 'of audience have travelled submitted report showed based upon documents about espionage in favour of Kingdom of'\n",
      "11: 0.995 : 'seen Mills accompanied by musicians as Browne Gibbons <OOV> <OOV> Lewis <OOV> <OOV> <OOV> <OOV>'\n",
      "12: 0.995 : '<OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> McCormack <OOV> <OOV> Moore <OOV>'\n",
      "13: 0.995 : '<OOV> <OOV> <OOV> Montana Sean <OOV> <OOV> Wayne Brown Khalifa <OOV> <OOV> Drake <OOV>'\n",
      "14: 0.995 : 'Russia Spain Kingdom Germany France Italy Austria Norway Poland Ukraine Belgium Bosnia Herzegovina Hungary'\n",
      "15: 0.995 : 'education to access agency abilities independence as issues to tied created <OOV> as poverty <OOV> issues of'\n",
      "16: 0.995 : 'has offices in Toronto Vancouver City California Caracas Jakarta Kong <OOV> Dhaka Singapore Casablanca'\n",
      "17: 0.995 : 'including <OOV> in nature of Corpus <OOV> Prohibition <OOV> with in territory for for'\n",
      "18: 0.995 : 'provides to students in communities of Wilson <OOV> <OOV> <OOV> <OOV> <OOV> Bassett <OOV>'\n",
      "19: 0.995 : 'of knight <OOV> of Order of Cross of Chamberlain Knight was knight of Order of'\n",
      "================================================================================\n",
      "(0:0:39) step 10/15121, epoch 0 Training Loss = 0.16484 :: 631.645 phrases/sec :: (8:39:38) hours left\n",
      "(0:1:10) step 20/15121, epoch 0 Training Loss = 0.16375 :: 594.907 phrases/sec :: (9:11:9) hours left\n",
      "(0:1:42) step 30/15121, epoch 0 Training Loss = 0.16455 :: 574.084 phrases/sec :: (9:30:35) hours left\n",
      "(0:2:10) step 40/15121, epoch 0 Training Loss = 0.16388 :: 588.590 phrases/sec :: (9:16:3) hours left\n",
      "(0:2:40) step 50/15121, epoch 0 Training Loss = 0.16383 :: 592.125 phrases/sec :: (9:12:14) hours left\n",
      "(0:3:5) step 60/15121, epoch 0 Training Loss = 0.16344 :: 609.872 phrases/sec :: (8:55:43) hours left\n",
      "(0:3:31) step 70/15121, epoch 0 Training Loss = 0.16552 :: 618.803 phrases/sec :: (8:47:31) hours left\n",
      "(0:3:58) step 80/15121, epoch 0 Training Loss = 0.16431 :: 623.339 phrases/sec :: (8:43:13) hours left\n",
      "(0:4:24) step 90/15121, epoch 0 Training Loss = 0.16554 :: 631.160 phrases/sec :: (8:36:17) hours left\n",
      "(0:4:48) step 100/15121, epoch 0 Training Loss = 0.16401 :: 641.757 phrases/sec :: (8:27:18) hours left\n",
      "(0:5:14) step 110/15121, epoch 0 Training Loss = 0.16443 :: 648.643 phrases/sec :: (8:21:27) hours left\n",
      "(0:5:41) step 120/15121, epoch 0 Training Loss = 0.16370 :: 651.417 phrases/sec :: (8:18:51) hours left\n",
      "(0:6:6) step 130/15121, epoch 0 Training Loss = 0.16394 :: 657.193 phrases/sec :: (8:14:0) hours left\n",
      "(0:6:34) step 140/15121, epoch 0 Training Loss = 0.16364 :: 659.593 phrases/sec :: (8:11:44) hours left\n",
      "(0:7:0) step 150/15121, epoch 0 Training Loss = 0.16340 :: 661.877 phrases/sec :: (8:9:35) hours left\n",
      "(0:7:28) step 160/15121, epoch 0 Training Loss = 0.16543 :: 662.286 phrases/sec :: (8:8:48) hours left\n",
      "(0:7:55) step 170/15121, epoch 0 Training Loss = 0.16458 :: 664.015 phrases/sec :: (8:7:4) hours left\n",
      "(0:8:24) step 180/15121, epoch 0 Training Loss = 0.16391 :: 662.902 phrases/sec :: (8:7:25) hours left\n",
      "(0:8:51) step 190/15121, epoch 0 Training Loss = 0.16482 :: 665.432 phrases/sec :: (8:5:6) hours left\n",
      "(0:9:18) step 200/15121, epoch 0 Training Loss = 0.16534 :: 666.598 phrases/sec :: (8:3:47) hours left\n",
      "(0:9:47) step 210/15121, epoch 0 Training Loss = 0.16389 :: 665.149 phrases/sec :: (8:4:22) hours left\n",
      "(0:10:14) step 220/15121, epoch 0 Training Loss = 0.16364 :: 666.765 phrases/sec :: (8:2:44) hours left\n",
      "(0:10:40) step 230/15121, epoch 0 Training Loss = 0.16380 :: 668.962 phrases/sec :: (8:0:40) hours left\n",
      "(0:11:9) step 240/15121, epoch 0 Training Loss = 0.16394 :: 667.441 phrases/sec :: (8:1:18) hours left\n",
      "(0:11:37) step 250/15121, epoch 0 Training Loss = 0.16336 :: 668.569 phrases/sec :: (8:0:1) hours left\n",
      "(0:12:4) step 260/15121, epoch 0 Training Loss = 0.16370 :: 669.887 phrases/sec :: (7:58:36) hours left\n",
      "(0:12:32) step 270/15121, epoch 0 Training Loss = 0.16369 :: 670.051 phrases/sec :: (7:58:1) hours left\n",
      "(0:12:59) step 280/15121, epoch 0 Training Loss = 0.16498 :: 671.168 phrases/sec :: (7:56:45) hours left\n",
      "(0:13:27) step 290/15121, epoch 0 Training Loss = 0.16424 :: 671.236 phrases/sec :: (7:56:14) hours left\n",
      "(0:13:54) step 300/15121, epoch 0 Training Loss = 0.16344 :: 672.403 phrases/sec :: (7:54:56) hours left\n",
      "(0:14:21) step 310/15121, epoch 0 Training Loss = 0.16366 :: 672.963 phrases/sec :: (7:54:4) hours left\n",
      "(0:14:48) step 320/15121, epoch 0 Training Loss = 0.16402 :: 674.404 phrases/sec :: (7:52:35) hours left\n",
      "(0:15:16) step 330/15121, epoch 0 Training Loss = 0.16454 :: 675.176 phrases/sec :: (7:51:35) hours left\n",
      "(0:15:45) step 340/15121, epoch 0 Training Loss = 0.16397 :: 677.025 phrases/sec :: (7:49:45) hours left\n",
      "(0:16:12) step 350/15121, epoch 0 Training Loss = 0.16393 :: 678.461 phrases/sec :: (7:48:17) hours left\n",
      "(0:16:39) step 360/15121, epoch 0 Training Loss = 0.16354 :: 679.380 phrases/sec :: (7:47:11) hours left\n",
      "(0:17:7) step 370/15121, epoch 0 Training Loss = 0.16490 :: 679.434 phrases/sec :: (7:46:40) hours left\n",
      "(0:17:34) step 380/15121, epoch 0 Training Loss = 0.16342 :: 680.572 phrases/sec :: (7:45:25) hours left\n",
      "(0:18:1) step 390/15121, epoch 0 Training Loss = 0.16367 :: 681.691 phrases/sec :: (7:44:11) hours left\n",
      "(0:18:27) step 400/15121, epoch 0 Training Loss = 0.16385 :: 683.063 phrases/sec :: (7:42:47) hours left\n",
      "(0:18:55) step 410/15121, epoch 0 Training Loss = 0.16395 :: 683.132 phrases/sec :: (7:42:16) hours left\n",
      "(0:19:23) step 420/15121, epoch 0 Training Loss = 0.16479 :: 682.827 phrases/sec :: (7:42:0) hours left\n",
      "(0:19:51) step 430/15121, epoch 0 Training Loss = 0.16396 :: 682.879 phrases/sec :: (7:41:30) hours left\n",
      "(0:20:20) step 440/15121, epoch 0 Training Loss = 0.16473 :: 682.306 phrases/sec :: (7:41:26) hours left\n",
      "(0:20:48) step 450/15121, epoch 0 Training Loss = 0.16368 :: 682.207 phrases/sec :: (7:41:2) hours left\n",
      "(0:21:16) step 460/15121, epoch 0 Training Loss = 0.16502 :: 681.819 phrases/sec :: (7:40:50) hours left\n",
      "(0:21:43) step 470/15121, epoch 0 Training Loss = 0.16466 :: 682.862 phrases/sec :: (7:39:39) hours left\n",
      "(0:22:11) step 480/15121, epoch 0 Training Loss = 0.16357 :: 682.558 phrases/sec :: (7:39:24) hours left\n",
      "(0:22:40) step 490/15121, epoch 0 Training Loss = 0.16405 :: 682.201 phrases/sec :: (7:39:10) hours left\n",
      "(0:23:7) step 500/15121, epoch 0 Training Loss = 0.16362 :: 682.696 phrases/sec :: (7:38:22) hours left\n",
      "(0:23:36) step 510/15121, epoch 0 Training Loss = 0.16378 :: 682.259 phrases/sec :: (7:38:12) hours left\n",
      "(0:24:5) step 520/15121, epoch 0 Training Loss = 0.16764 :: 681.352 phrases/sec :: (7:38:21) hours left\n",
      "(0:24:33) step 530/15121, epoch 0 Training Loss = 0.16350 :: 681.231 phrases/sec :: (7:37:58) hours left\n",
      "(0:25:1) step 540/15121, epoch 0 Training Loss = 0.16447 :: 681.051 phrases/sec :: (7:37:38) hours left\n",
      "(0:25:31) step 550/15121, epoch 0 Training Loss = 0.16318 :: 680.110 phrases/sec :: (7:37:48) hours left\n",
      "(0:25:57) step 560/15121, epoch 0 Training Loss = 0.16470 :: 681.092 phrases/sec :: (7:36:40) hours left\n",
      "(0:26:26) step 570/15121, epoch 0 Training Loss = 0.16359 :: 680.642 phrases/sec :: (7:36:30) hours left\n",
      "(0:26:56) step 580/15121, epoch 0 Training Loss = 0.16341 :: 679.423 phrases/sec :: (7:36:52) hours left\n",
      "(0:27:24) step 590/15121, epoch 0 Training Loss = 0.16386 :: 679.473 phrases/sec :: (7:36:22) hours left\n",
      "(0:27:52) step 600/15121, epoch 0 Training Loss = 0.16586 :: 679.670 phrases/sec :: (7:35:45) hours left\n",
      "(0:28:21) step 610/15121, epoch 0 Training Loss = 0.16403 :: 679.584 phrases/sec :: (7:35:21) hours left\n",
      "(0:28:47) step 620/15121, epoch 0 Training Loss = 0.16361 :: 680.468 phrases/sec :: (7:34:16) hours left\n",
      "(0:29:16) step 630/15121, epoch 0 Training Loss = 0.16351 :: 680.029 phrases/sec :: (7:34:6) hours left\n",
      "(0:29:44) step 640/15121, epoch 0 Training Loss = 0.16390 :: 680.095 phrases/sec :: (7:33:36) hours left\n",
      "Saving model...\n",
      "(0:30:14) step 650/15121, epoch 0 Training Loss = 0.16444 :: 680.089 phrases/sec :: (7:33:5) hours left\n",
      "(0:30:42) step 660/15121, epoch 0 Training Loss = 0.16361 :: 680.495 phrases/sec :: (7:32:21) hours left\n",
      "(0:31:9) step 670/15121, epoch 0 Training Loss = 0.16374 :: 680.964 phrases/sec :: (7:31:34) hours left\n",
      "(0:31:37) step 680/15121, epoch 0 Training Loss = 0.16460 :: 680.981 phrases/sec :: (7:31:5) hours left\n",
      "(0:32:3) step 690/15121, epoch 0 Training Loss = 0.16367 :: 681.763 phrases/sec :: (7:30:5) hours left\n",
      "(0:32:31) step 700/15121, epoch 0 Training Loss = 0.16405 :: 681.813 phrases/sec :: (7:29:35) hours left\n",
      "(0:32:58) step 710/15121, epoch 0 Training Loss = 0.16348 :: 682.509 phrases/sec :: (7:28:39) hours left\n",
      "(0:33:26) step 720/15121, epoch 0 Training Loss = 0.16405 :: 682.338 phrases/sec :: (7:28:18) hours left\n",
      "(0:33:54) step 730/15121, epoch 0 Training Loss = 0.16531 :: 682.271 phrases/sec :: (7:27:53) hours left\n",
      "(0:34:22) step 740/15121, epoch 0 Training Loss = 0.17874 :: 682.256 phrases/sec :: (7:27:26) hours left\n",
      "(0:34:50) step 750/15121, epoch 0 Training Loss = 0.16774 :: 682.438 phrases/sec :: (7:26:51) hours left\n",
      "(0:35:18) step 760/15121, epoch 0 Training Loss = 0.16421 :: 682.544 phrases/sec :: (7:26:18) hours left\n",
      "(0:35:45) step 770/15121, epoch 0 Training Loss = 0.16458 :: 682.836 phrases/sec :: (7:25:38) hours left\n",
      "(0:36:12) step 780/15121, epoch 0 Training Loss = 0.16355 :: 683.086 phrases/sec :: (7:25:1) hours left\n",
      "(0:36:40) step 790/15121, epoch 0 Training Loss = 0.16452 :: 683.346 phrases/sec :: (7:24:22) hours left\n",
      "(0:37:7) step 800/15121, epoch 0 Training Loss = 0.16346 :: 683.488 phrases/sec :: (7:23:49) hours left\n",
      "(0:37:37) step 810/15121, epoch 0 Training Loss = 0.16385 :: 682.913 phrases/sec :: (7:23:43) hours left\n",
      "(0:38:4) step 820/15121, epoch 0 Training Loss = 0.16354 :: 683.240 phrases/sec :: (7:23:2) hours left\n",
      "(0:38:31) step 830/15121, epoch 0 Training Loss = 0.16551 :: 683.579 phrases/sec :: (7:22:21) hours left\n",
      "(0:38:58) step 840/15121, epoch 0 Training Loss = 0.16398 :: 683.805 phrases/sec :: (7:21:44) hours left\n",
      "(0:39:26) step 850/15121, epoch 0 Training Loss = 0.16419 :: 683.808 phrases/sec :: (7:21:16) hours left\n",
      "(0:39:53) step 860/15121, epoch 0 Training Loss = 0.16343 :: 684.333 phrases/sec :: (7:20:27) hours left\n",
      "(0:40:21) step 870/15121, epoch 0 Training Loss = 0.16381 :: 684.416 phrases/sec :: (7:19:56) hours left\n",
      "(0:40:49) step 880/15121, epoch 0 Training Loss = 0.16440 :: 684.519 phrases/sec :: (7:19:24) hours left\n",
      "(0:41:17) step 890/15121, epoch 0 Training Loss = 0.16391 :: 684.313 phrases/sec :: (7:19:4) hours left\n",
      "(0:41:44) step 900/15121, epoch 0 Training Loss = 0.16487 :: 684.572 phrases/sec :: (7:18:26) hours left\n",
      "(0:42:12) step 910/15121, epoch 0 Training Loss = 0.16361 :: 684.613 phrases/sec :: (7:17:57) hours left\n",
      "(0:42:41) step 920/15121, epoch 0 Training Loss = 0.16380 :: 684.250 phrases/sec :: (7:17:43) hours left\n",
      "(0:43:8) step 930/15121, epoch 0 Training Loss = 0.16385 :: 684.541 phrases/sec :: (7:17:4) hours left\n",
      "(0:43:36) step 940/15121, epoch 0 Training Loss = 0.16380 :: 684.580 phrases/sec :: (7:16:34) hours left\n",
      "(0:44:3) step 950/15121, epoch 0 Training Loss = 0.16345 :: 684.681 phrases/sec :: (7:16:3) hours left\n",
      "(0:44:31) step 960/15121, epoch 0 Training Loss = 0.16356 :: 684.850 phrases/sec :: (7:15:28) hours left\n",
      "(0:44:58) step 970/15121, epoch 0 Training Loss = 0.17015 :: 685.209 phrases/sec :: (7:14:46) hours left\n",
      "(0:45:25) step 980/15121, epoch 0 Training Loss = 0.16374 :: 685.290 phrases/sec :: (7:14:15) hours left\n",
      "(0:45:52) step 990/15121, epoch 0 Training Loss = 0.16379 :: 685.719 phrases/sec :: (7:13:31) hours left\n",
      "(0:46:19) step 1000/15121, epoch 0 Training Loss = 0.16349 :: 685.887 phrases/sec :: (7:12:56) hours left\n",
      "(0:46:46) step 1010/15121, epoch 0 Training Loss = 0.16377 :: 686.064 phrases/sec :: (7:12:22) hours left\n",
      "(0:47:14) step 1020/15121, epoch 0 Training Loss = 0.16322 :: 686.408 phrases/sec :: (7:11:40) hours left\n",
      "(0:47:42) step 1030/15121, epoch 0 Training Loss = 0.16391 :: 686.046 phrases/sec :: (7:11:26) hours left\n",
      "(0:48:10) step 1040/15121, epoch 0 Training Loss = 0.16390 :: 685.997 phrases/sec :: (7:11:0) hours left\n",
      "(0:48:38) step 1050/15121, epoch 0 Training Loss = 0.17465 :: 686.107 phrases/sec :: (7:10:29) hours left\n",
      "(0:49:4) step 1060/15121, epoch 0 Training Loss = 0.16353 :: 686.706 phrases/sec :: (7:9:37) hours left\n",
      "(0:49:31) step 1070/15121, epoch 0 Training Loss = 0.16382 :: 686.846 phrases/sec :: (7:9:4) hours left\n",
      "(0:49:59) step 1080/15121, epoch 0 Training Loss = 0.16379 :: 686.718 phrases/sec :: (7:8:41) hours left\n",
      "(0:50:27) step 1090/15121, epoch 0 Training Loss = 0.16327 :: 686.708 phrases/sec :: (7:8:14) hours left\n",
      "(0:50:53) step 1100/15121, epoch 0 Training Loss = 0.16378 :: 687.187 phrases/sec :: (7:7:28) hours left\n",
      "(0:51:22) step 1110/15121, epoch 0 Training Loss = 0.16343 :: 686.903 phrases/sec :: (7:7:11) hours left\n",
      "(0:51:50) step 1120/15121, epoch 0 Training Loss = 0.16583 :: 686.848 phrases/sec :: (7:6:45) hours left\n",
      "(0:52:19) step 1130/15121, epoch 0 Training Loss = 0.16550 :: 686.661 phrases/sec :: (7:6:25) hours left\n",
      "(0:52:47) step 1140/15121, epoch 0 Training Loss = 0.16596 :: 686.513 phrases/sec :: (7:6:3) hours left\n",
      "(0:53:15) step 1150/15121, epoch 0 Training Loss = 0.16431 :: 686.497 phrases/sec :: (7:5:35) hours left\n",
      "(0:53:42) step 1160/15121, epoch 0 Training Loss = 0.16358 :: 686.622 phrases/sec :: (7:5:3) hours left\n",
      "(0:54:9) step 1170/15121, epoch 0 Training Loss = 0.16351 :: 686.829 phrases/sec :: (7:4:27) hours left\n",
      "(0:54:38) step 1180/15121, epoch 0 Training Loss = 0.16469 :: 686.458 phrases/sec :: (7:4:14) hours left\n",
      "(0:55:5) step 1190/15121, epoch 0 Training Loss = 0.16516 :: 686.814 phrases/sec :: (7:3:32) hours left\n",
      "(0:55:31) step 1200/15121, epoch 0 Training Loss = 0.16376 :: 687.168 phrases/sec :: (7:2:51) hours left\n",
      "(0:55:58) step 1210/15121, epoch 0 Training Loss = 0.16402 :: 687.385 phrases/sec :: (7:2:15) hours left\n",
      "(0:56:26) step 1220/15121, epoch 0 Training Loss = 0.16488 :: 687.568 phrases/sec :: (7:1:40) hours left\n",
      "(0:56:53) step 1230/15121, epoch 0 Training Loss = 0.16405 :: 687.515 phrases/sec :: (7:1:14) hours left\n",
      "(0:57:22) step 1240/15121, epoch 0 Training Loss = 0.16384 :: 687.398 phrases/sec :: (7:0:51) hours left\n",
      "(0:57:51) step 1250/15121, epoch 0 Training Loss = 0.16388 :: 687.035 phrases/sec :: (7:0:37) hours left\n",
      "(0:58:18) step 1260/15121, epoch 0 Training Loss = 0.16468 :: 687.161 phrases/sec :: (7:0:4) hours left\n",
      "(0:58:46) step 1270/15121, epoch 0 Training Loss = 0.16362 :: 687.004 phrases/sec :: (6:59:42) hours left\n",
      "(0:59:15) step 1280/15121, epoch 0 Training Loss = 0.16385 :: 686.781 phrases/sec :: (6:59:23) hours left\n",
      "(0:59:43) step 1290/15121, epoch 0 Training Loss = 0.16326 :: 686.828 phrases/sec :: (6:58:54) hours left\n",
      "Saving model...\n",
      "(1:0:12) step 1300/15121, epoch 0 Training Loss = 0.16441 :: 687.072 phrases/sec :: (6:58:14) hours left\n",
      "(1:0:39) step 1310/15121, epoch 0 Training Loss = 0.16406 :: 687.109 phrases/sec :: (6:57:45) hours left\n",
      "(1:1:7) step 1320/15121, epoch 0 Training Loss = 0.16386 :: 687.077 phrases/sec :: (6:57:18) hours left\n",
      "(1:1:35) step 1330/15121, epoch 0 Training Loss = 0.16350 :: 687.250 phrases/sec :: (6:56:44) hours left\n",
      "(1:2:2) step 1340/15121, epoch 0 Training Loss = 0.16615 :: 687.387 phrases/sec :: (6:56:11) hours left\n",
      "(1:2:29) step 1350/15121, epoch 0 Training Loss = 0.16348 :: 687.463 phrases/sec :: (6:55:40) hours left\n",
      "(1:2:58) step 1360/15121, epoch 0 Training Loss = 0.17004 :: 687.347 phrases/sec :: (6:55:17) hours left\n",
      "(1:3:24) step 1370/15121, epoch 0 Training Loss = 0.16426 :: 687.673 phrases/sec :: (6:54:37) hours left\n",
      "(1:3:52) step 1380/15121, epoch 0 Training Loss = 0.16704 :: 687.607 phrases/sec :: (6:54:11) hours left\n",
      "(1:4:20) step 1390/15121, epoch 0 Training Loss = 0.16371 :: 687.770 phrases/sec :: (6:53:37) hours left\n",
      "(1:4:47) step 1400/15121, epoch 0 Training Loss = 0.16381 :: 687.722 phrases/sec :: (6:53:11) hours left\n",
      "(1:5:15) step 1410/15121, epoch 0 Training Loss = 0.16406 :: 687.750 phrases/sec :: (6:52:42) hours left\n",
      "(1:5:44) step 1420/15121, epoch 0 Training Loss = 0.16367 :: 687.615 phrases/sec :: (6:52:20) hours left\n",
      "(1:6:12) step 1430/15121, epoch 0 Training Loss = 0.16341 :: 687.577 phrases/sec :: (6:51:53) hours left\n",
      "(1:6:39) step 1440/15121, epoch 0 Training Loss = 0.16360 :: 687.621 phrases/sec :: (6:51:24) hours left\n",
      "(1:7:6) step 1450/15121, epoch 0 Training Loss = 0.16325 :: 687.818 phrases/sec :: (6:50:49) hours left\n",
      "(1:7:34) step 1460/15121, epoch 0 Training Loss = 0.16440 :: 687.882 phrases/sec :: (6:50:18) hours left\n",
      "(1:8:1) step 1470/15121, epoch 0 Training Loss = 0.16468 :: 687.913 phrases/sec :: (6:49:50) hours left\n",
      "(1:8:30) step 1480/15121, epoch 0 Training Loss = 0.16404 :: 687.764 phrases/sec :: (6:49:27) hours left\n",
      "(1:8:56) step 1490/15121, epoch 0 Training Loss = 0.16353 :: 688.192 phrases/sec :: (6:48:44) hours left\n",
      "(1:9:24) step 1500/15121, epoch 0 Training Loss = 0.16422 :: 688.172 phrases/sec :: (6:48:17) hours left\n",
      "(1:9:51) step 1510/15121, epoch 0 Training Loss = 0.16475 :: 688.221 phrases/sec :: (6:47:47) hours left\n",
      "(1:10:18) step 1520/15121, epoch 0 Training Loss = 0.16357 :: 688.387 phrases/sec :: (6:47:13) hours left\n",
      "(1:10:46) step 1530/15121, epoch 0 Training Loss = 0.16336 :: 688.389 phrases/sec :: (6:46:45) hours left\n",
      "(1:11:13) step 1540/15121, epoch 0 Training Loss = 0.16341 :: 688.469 phrases/sec :: (6:46:15) hours left\n",
      "(1:11:41) step 1550/15121, epoch 0 Training Loss = 0.16505 :: 688.455 phrases/sec :: (6:45:47) hours left\n",
      "(1:12:9) step 1560/15121, epoch 0 Training Loss = 0.16464 :: 688.559 phrases/sec :: (6:45:16) hours left\n",
      "(1:12:36) step 1570/15121, epoch 0 Training Loss = 0.16524 :: 688.739 phrases/sec :: (6:44:41) hours left\n",
      "(1:13:3) step 1580/15121, epoch 0 Training Loss = 0.16438 :: 688.742 phrases/sec :: (6:44:13) hours left\n",
      "(1:13:31) step 1590/15121, epoch 0 Training Loss = 0.16383 :: 688.779 phrases/sec :: (6:43:44) hours left\n",
      "(1:14:2) step 1600/15121, epoch 0 Training Loss = 0.16338 :: 688.191 phrases/sec :: (6:43:38) hours left\n",
      "(1:14:29) step 1610/15121, epoch 0 Training Loss = 0.16357 :: 688.272 phrases/sec :: (6:43:7) hours left\n",
      "(1:14:58) step 1620/15121, epoch 0 Training Loss = 0.16518 :: 688.069 phrases/sec :: (6:42:47) hours left\n",
      "(1:15:26) step 1630/15121, epoch 0 Training Loss = 0.16547 :: 687.960 phrases/sec :: (6:42:23) hours left\n",
      "(1:15:54) step 1640/15121, epoch 0 Training Loss = 0.16677 :: 687.978 phrases/sec :: (6:41:54) hours left\n",
      "(1:16:23) step 1650/15121, epoch 0 Training Loss = 0.16419 :: 687.626 phrases/sec :: (6:41:40) hours left\n",
      "(1:16:51) step 1660/15121, epoch 0 Training Loss = 0.16386 :: 687.632 phrases/sec :: (6:41:12) hours left\n",
      "(1:17:18) step 1670/15121, epoch 0 Training Loss = 0.16361 :: 687.786 phrases/sec :: (6:40:38) hours left\n",
      "(1:17:44) step 1680/15121, epoch 0 Training Loss = 0.16360 :: 688.293 phrases/sec :: (6:39:51) hours left\n",
      "(1:18:13) step 1690/15121, epoch 0 Training Loss = 0.16308 :: 687.985 phrases/sec :: (6:39:35) hours left\n",
      "(1:18:41) step 1700/15121, epoch 0 Training Loss = 0.16505 :: 687.894 phrases/sec :: (6:39:11) hours left\n",
      "(1:19:9) step 1710/15121, epoch 0 Training Loss = 0.16399 :: 687.860 phrases/sec :: (6:38:44) hours left\n",
      "(1:19:37) step 1720/15121, epoch 0 Training Loss = 0.16312 :: 687.768 phrases/sec :: (6:38:20) hours left\n",
      "(1:20:7) step 1730/15121, epoch 0 Training Loss = 0.16398 :: 687.524 phrases/sec :: (6:38:0) hours left\n",
      "(1:20:44) step 1740/15121, epoch 0 Training Loss = 0.16312 :: 686.316 phrases/sec :: (6:38:14) hours left\n",
      "(1:21:19) step 1750/15121, epoch 0 Training Loss = 0.16334 :: 685.654 phrases/sec :: (6:38:6) hours left\n",
      "(1:21:50) step 1760/15121, epoch 0 Training Loss = 0.16346 :: 685.516 phrases/sec :: (6:37:41) hours left\n",
      "(1:22:17) step 1770/15121, epoch 0 Training Loss = 0.16330 :: 685.719 phrases/sec :: (6:37:6) hours left\n",
      "(1:22:45) step 1780/15121, epoch 0 Training Loss = 0.16359 :: 685.625 phrases/sec :: (6:36:41) hours left\n",
      "(1:23:12) step 1790/15121, epoch 0 Training Loss = 0.16461 :: 685.840 phrases/sec :: (6:36:5) hours left\n",
      "(1:23:40) step 1800/15121, epoch 0 Training Loss = 0.16316 :: 685.840 phrases/sec :: (6:35:37) hours left\n",
      "(1:24:7) step 1810/15121, epoch 0 Training Loss = 0.16359 :: 685.990 phrases/sec :: (6:35:4) hours left\n",
      "(1:24:34) step 1820/15121, epoch 0 Training Loss = 0.16355 :: 686.187 phrases/sec :: (6:34:28) hours left\n",
      "(1:25:4) step 1830/15121, epoch 0 Training Loss = 0.16363 :: 685.925 phrases/sec :: (6:34:10) hours left\n",
      "(1:25:32) step 1840/15121, epoch 0 Training Loss = 0.16369 :: 685.843 phrases/sec :: (6:33:45) hours left\n",
      "(1:26:1) step 1850/15121, epoch 0 Training Loss = 0.16383 :: 685.672 phrases/sec :: (6:33:23) hours left\n",
      "(1:26:32) step 1860/15121, epoch 0 Training Loss = 0.16375 :: 685.205 phrases/sec :: (6:33:12) hours left\n",
      "(1:27:0) step 1870/15121, epoch 0 Training Loss = 0.16357 :: 685.210 phrases/sec :: (6:32:44) hours left\n",
      "(1:27:28) step 1880/15121, epoch 0 Training Loss = 0.16358 :: 685.136 phrases/sec :: (6:32:18) hours left\n",
      "(1:27:57) step 1890/15121, epoch 0 Training Loss = 0.16371 :: 684.984 phrases/sec :: (6:31:56) hours left\n",
      "(1:28:27) step 1900/15121, epoch 0 Training Loss = 0.16333 :: 684.741 phrases/sec :: (6:31:37) hours left\n",
      "(1:29:1) step 1910/15121, epoch 0 Training Loss = 0.16365 :: 683.812 phrases/sec :: (6:31:42) hours left\n",
      "(1:29:30) step 1920/15121, epoch 0 Training Loss = 0.16376 :: 683.736 phrases/sec :: (6:31:16) hours left\n",
      "(1:29:58) step 1930/15121, epoch 0 Training Loss = 0.16499 :: 683.676 phrases/sec :: (6:30:50) hours left\n",
      "Saving model...\n",
      "(1:30:26) step 1940/15121, epoch 0 Training Loss = 0.16365 :: 683.890 phrases/sec :: (6:30:13) hours left\n",
      "(1:30:49) step 1950/15121, epoch 0 Training Loss = 0.16363 :: 683.871 phrases/sec :: (6:29:51) hours left\n",
      "(1:31:12) step 1960/15121, epoch 0 Training Loss = 0.16350 :: 683.917 phrases/sec :: (6:29:26) hours left\n",
      "(1:31:35) step 1970/15121, epoch 0 Training Loss = 0.16333 :: 684.022 phrases/sec :: (6:28:59) hours left\n",
      "(1:31:59) step 1980/15121, epoch 0 Training Loss = 0.16363 :: 683.964 phrases/sec :: (6:28:37) hours left\n",
      "(1:32:25) step 1990/15121, epoch 0 Training Loss = 0.16383 :: 683.775 phrases/sec :: (6:28:19) hours left\n",
      "(1:32:50) step 2000/15121, epoch 0 Training Loss = 0.16640 :: 683.632 phrases/sec :: (6:28:0) hours left\n",
      "(1:33:16) step 2010/15121, epoch 0 Training Loss = 0.16382 :: 683.478 phrases/sec :: (6:27:40) hours left\n",
      "(1:33:41) step 2020/15121, epoch 0 Training Loss = 0.16350 :: 683.606 phrases/sec :: (6:27:10) hours left\n",
      "(1:34:6) step 2030/15121, epoch 0 Training Loss = 0.16489 :: 683.645 phrases/sec :: (6:26:43) hours left\n",
      "(1:34:31) step 2040/15121, epoch 0 Training Loss = 0.16343 :: 683.785 phrases/sec :: (6:26:12) hours left\n",
      "(1:34:56) step 2050/15121, epoch 0 Training Loss = 0.16361 :: 683.884 phrases/sec :: (6:25:43) hours left\n",
      "(1:35:22) step 2060/15121, epoch 0 Training Loss = 0.16342 :: 684.100 phrases/sec :: (6:25:9) hours left\n",
      "(1:35:48) step 2070/15121, epoch 0 Training Loss = 0.16385 :: 684.005 phrases/sec :: (6:24:46) hours left\n",
      "(1:36:14) step 2080/15121, epoch 0 Training Loss = 0.16347 :: 684.152 phrases/sec :: (6:24:14) hours left\n",
      "(1:36:41) step 2090/15121, epoch 0 Training Loss = 0.16349 :: 684.122 phrases/sec :: (6:23:49) hours left\n",
      "(1:37:7) step 2100/15121, epoch 0 Training Loss = 0.16390 :: 684.134 phrases/sec :: (6:23:21) hours left\n",
      "(1:37:34) step 2110/15121, epoch 0 Training Loss = 0.16322 :: 684.090 phrases/sec :: (6:22:56) hours left\n",
      "(1:38:2) step 2120/15121, epoch 0 Training Loss = 0.16327 :: 684.008 phrases/sec :: (6:22:32) hours left\n",
      "(1:38:29) step 2130/15121, epoch 0 Training Loss = 0.16593 :: 684.047 phrases/sec :: (6:22:4) hours left\n",
      "(1:38:56) step 2140/15121, epoch 0 Training Loss = 0.16385 :: 684.080 phrases/sec :: (6:21:36) hours left\n",
      "(1:39:21) step 2150/15121, epoch 0 Training Loss = 0.17091 :: 684.363 phrases/sec :: (6:20:58) hours left\n",
      "(1:39:48) step 2160/15121, epoch 0 Training Loss = 0.16335 :: 684.443 phrases/sec :: (6:20:28) hours left\n",
      "(1:40:14) step 2170/15121, epoch 0 Training Loss = 0.16357 :: 684.579 phrases/sec :: (6:19:56) hours left\n",
      "(1:40:42) step 2180/15121, epoch 0 Training Loss = 0.16535 :: 684.585 phrases/sec :: (6:19:28) hours left\n",
      "(1:41:7) step 2190/15121, epoch 0 Training Loss = 0.16386 :: 684.832 phrases/sec :: (6:18:52) hours left\n",
      "(1:41:36) step 2200/15121, epoch 0 Training Loss = 0.16530 :: 684.597 phrases/sec :: (6:18:33) hours left\n",
      "(1:42:4) step 2210/15121, epoch 0 Training Loss = 0.16403 :: 684.588 phrases/sec :: (6:18:6) hours left\n",
      "(1:42:31) step 2220/15121, epoch 0 Training Loss = 0.16347 :: 684.709 phrases/sec :: (6:17:34) hours left\n",
      "(1:42:56) step 2230/15121, epoch 0 Training Loss = 0.16428 :: 685.023 phrases/sec :: (6:16:55) hours left\n",
      "(1:43:24) step 2240/15121, epoch 0 Training Loss = 0.16325 :: 685.064 phrases/sec :: (6:16:26) hours left\n",
      "(1:43:51) step 2250/15121, epoch 0 Training Loss = 0.16357 :: 685.111 phrases/sec :: (6:15:57) hours left\n",
      "(1:44:19) step 2260/15121, epoch 0 Training Loss = 0.16357 :: 685.100 phrases/sec :: (6:15:29) hours left\n",
      "(1:44:46) step 2270/15121, epoch 0 Training Loss = 0.16336 :: 685.202 phrases/sec :: (6:14:58) hours left\n",
      "(1:45:13) step 2280/15121, epoch 0 Training Loss = 0.16377 :: 685.297 phrases/sec :: (6:14:27) hours left\n",
      "(1:45:41) step 2290/15121, epoch 0 Training Loss = 0.16421 :: 685.284 phrases/sec :: (6:13:59) hours left\n",
      "(1:46:9) step 2300/15121, epoch 0 Training Loss = 0.16360 :: 685.237 phrases/sec :: (6:13:33) hours left\n",
      "(1:46:36) step 2310/15121, epoch 0 Training Loss = 0.16582 :: 685.395 phrases/sec :: (6:13:0) hours left\n",
      "(1:47:5) step 2320/15121, epoch 0 Training Loss = 0.16378 :: 685.295 phrases/sec :: (6:12:35) hours left\n",
      "(1:47:33) step 2330/15121, epoch 0 Training Loss = 0.16331 :: 685.199 phrases/sec :: (6:12:11) hours left\n",
      "(1:48:0) step 2340/15121, epoch 0 Training Loss = 0.16334 :: 685.339 phrases/sec :: (6:11:38) hours left\n",
      "(1:48:28) step 2350/15121, epoch 0 Training Loss = 0.16425 :: 685.445 phrases/sec :: (6:11:6) hours left\n",
      "(1:48:56) step 2360/15121, epoch 0 Training Loss = 0.16337 :: 685.316 phrases/sec :: (6:10:43) hours left\n",
      "(1:49:24) step 2370/15121, epoch 0 Training Loss = 0.16493 :: 685.410 phrases/sec :: (6:10:12) hours left\n",
      "(1:49:51) step 2380/15121, epoch 0 Training Loss = 0.16387 :: 685.561 phrases/sec :: (6:9:38) hours left\n",
      "(1:50:19) step 2390/15121, epoch 0 Training Loss = 0.16337 :: 685.503 phrases/sec :: (6:9:12) hours left\n",
      "(1:50:46) step 2400/15121, epoch 0 Training Loss = 0.16603 :: 685.639 phrases/sec :: (6:8:39) hours left\n",
      "(1:51:13) step 2410/15121, epoch 0 Training Loss = 0.16362 :: 685.855 phrases/sec :: (6:8:4) hours left\n",
      "(1:51:40) step 2420/15121, epoch 0 Training Loss = 0.16507 :: 685.936 phrases/sec :: (6:7:33) hours left\n",
      "(1:52:8) step 2430/15121, epoch 0 Training Loss = 0.16330 :: 685.940 phrases/sec :: (6:7:5) hours left\n",
      "(1:52:36) step 2440/15121, epoch 0 Training Loss = 0.16390 :: 685.913 phrases/sec :: (6:6:38) hours left\n",
      "(1:53:5) step 2450/15121, epoch 0 Training Loss = 0.16318 :: 685.863 phrases/sec :: (6:6:12) hours left\n",
      "(1:53:31) step 2460/15121, epoch 0 Training Loss = 0.16406 :: 686.037 phrases/sec :: (6:5:38) hours left\n",
      "(1:54:0) step 2470/15121, epoch 0 Training Loss = 0.16349 :: 685.960 phrases/sec :: (6:5:12) hours left\n",
      "(1:54:29) step 2480/15121, epoch 0 Training Loss = 0.16414 :: 685.793 phrases/sec :: (6:4:50) hours left\n",
      "(1:54:58) step 2490/15121, epoch 0 Training Loss = 0.16553 :: 685.691 phrases/sec :: (6:4:26) hours left\n",
      "(1:55:26) step 2500/15121, epoch 0 Training Loss = 0.16319 :: 685.600 phrases/sec :: (6:4:1) hours left\n",
      "(1:55:56) step 2510/15121, epoch 0 Training Loss = 0.16371 :: 685.355 phrases/sec :: (6:3:42) hours left\n",
      "(1:56:24) step 2520/15121, epoch 0 Training Loss = 0.16419 :: 685.320 phrases/sec :: (6:3:15) hours left\n",
      "(1:56:52) step 2530/15121, epoch 0 Training Loss = 0.16392 :: 685.311 phrases/sec :: (6:2:47) hours left\n",
      "(1:57:21) step 2540/15121, epoch 0 Training Loss = 0.16329 :: 685.151 phrases/sec :: (6:2:25) hours left\n",
      "(1:57:51) step 2550/15121, epoch 0 Training Loss = 0.16365 :: 684.928 phrases/sec :: (6:2:5) hours left\n",
      "(1:58:20) step 2560/15121, epoch 0 Training Loss = 0.16445 :: 684.721 phrases/sec :: (6:1:44) hours left\n",
      "(1:58:47) step 2570/15121, epoch 0 Training Loss = 0.16454 :: 684.846 phrases/sec :: (6:1:12) hours left\n",
      "(1:59:16) step 2580/15121, epoch 0 Training Loss = 0.16380 :: 684.782 phrases/sec :: (6:0:46) hours left\n",
      "(1:59:44) step 2590/15121, epoch 0 Training Loss = 0.16511 :: 684.709 phrases/sec :: (6:0:21) hours left\n",
      "(2:0:11) step 2600/15121, epoch 0 Training Loss = 0.16334 :: 684.832 phrases/sec :: (5:59:48) hours left\n",
      "Saving model...\n",
      "(2:0:41) step 2610/15121, epoch 0 Training Loss = 0.16362 :: 684.958 phrases/sec :: (5:59:14) hours left\n",
      "(2:1:8) step 2620/15121, epoch 0 Training Loss = 0.16369 :: 685.072 phrases/sec :: (5:58:42) hours left\n",
      "(2:1:37) step 2630/15121, epoch 0 Training Loss = 0.16415 :: 684.842 phrases/sec :: (5:58:22) hours left\n",
      "(2:2:6) step 2640/15121, epoch 0 Training Loss = 0.16354 :: 684.686 phrases/sec :: (5:57:59) hours left\n",
      "(2:2:35) step 2650/15121, epoch 0 Training Loss = 0.16492 :: 684.561 phrases/sec :: (5:57:36) hours left\n",
      "(2:3:4) step 2660/15121, epoch 0 Training Loss = 0.16381 :: 684.465 phrases/sec :: (5:57:11) hours left\n",
      "(2:3:32) step 2670/15121, epoch 0 Training Loss = 0.16818 :: 684.434 phrases/sec :: (5:56:44) hours left\n",
      "(2:4:0) step 2680/15121, epoch 0 Training Loss = 0.16368 :: 684.521 phrases/sec :: (5:56:13) hours left\n",
      "(2:4:29) step 2690/15121, epoch 0 Training Loss = 0.16471 :: 684.382 phrases/sec :: (5:55:50) hours left\n",
      "(2:4:57) step 2700/15121, epoch 0 Training Loss = 0.16329 :: 684.411 phrases/sec :: (5:55:21) hours left\n",
      "(2:5:26) step 2710/15121, epoch 0 Training Loss = 0.16387 :: 684.278 phrases/sec :: (5:54:57) hours left\n",
      "(2:5:54) step 2720/15121, epoch 0 Training Loss = 0.16351 :: 684.295 phrases/sec :: (5:54:28) hours left\n",
      "(2:6:23) step 2730/15121, epoch 0 Training Loss = 0.16321 :: 684.148 phrases/sec :: (5:54:6) hours left\n",
      "(2:6:51) step 2740/15121, epoch 0 Training Loss = 0.16417 :: 684.148 phrases/sec :: (5:53:37) hours left\n",
      "(2:7:18) step 2750/15121, epoch 0 Training Loss = 0.16359 :: 684.303 phrases/sec :: (5:53:4) hours left\n",
      "(2:7:45) step 2760/15121, epoch 0 Training Loss = 0.16395 :: 684.387 phrases/sec :: (5:52:33) hours left\n",
      "(2:8:13) step 2770/15121, epoch 0 Training Loss = 0.16398 :: 684.369 phrases/sec :: (5:52:6) hours left\n",
      "(2:8:42) step 2780/15121, epoch 0 Training Loss = 0.16379 :: 684.309 phrases/sec :: (5:51:39) hours left\n",
      "(2:9:10) step 2790/15121, epoch 0 Training Loss = 0.16409 :: 684.373 phrases/sec :: (5:51:9) hours left\n",
      "(2:9:38) step 2800/15121, epoch 0 Training Loss = 0.16330 :: 684.349 phrases/sec :: (5:50:41) hours left\n",
      "(2:10:5) step 2810/15121, epoch 0 Training Loss = 0.16346 :: 684.501 phrases/sec :: (5:50:8) hours left\n",
      "(2:10:32) step 2820/15121, epoch 0 Training Loss = 0.16378 :: 684.669 phrases/sec :: (5:49:34) hours left\n",
      "(2:10:59) step 2830/15121, epoch 0 Training Loss = 0.16357 :: 684.794 phrases/sec :: (5:49:2) hours left\n",
      "(2:11:27) step 2840/15121, epoch 0 Training Loss = 0.16330 :: 684.783 phrases/sec :: (5:48:34) hours left\n",
      "(2:11:54) step 2850/15121, epoch 0 Training Loss = 0.16329 :: 684.968 phrases/sec :: (5:48:0) hours left\n",
      "(2:12:22) step 2860/15121, epoch 0 Training Loss = 0.16360 :: 685.004 phrases/sec :: (5:47:31) hours left\n",
      "(2:12:50) step 2870/15121, epoch 0 Training Loss = 0.16333 :: 684.927 phrases/sec :: (5:47:5) hours left\n",
      "(2:13:18) step 2880/15121, epoch 0 Training Loss = 0.16486 :: 685.027 phrases/sec :: (5:46:34) hours left\n",
      "(2:13:45) step 2890/15121, epoch 0 Training Loss = 0.16378 :: 685.136 phrases/sec :: (5:46:2) hours left\n",
      "(2:14:12) step 2900/15121, epoch 0 Training Loss = 0.16348 :: 685.214 phrases/sec :: (5:45:31) hours left\n",
      "(2:14:40) step 2910/15121, epoch 0 Training Loss = 0.16369 :: 685.215 phrases/sec :: (5:45:3) hours left\n",
      "(2:15:9) step 2920/15121, epoch 0 Training Loss = 0.16337 :: 685.181 phrases/sec :: (5:44:36) hours left\n",
      "(2:15:36) step 2930/15121, epoch 0 Training Loss = 0.16367 :: 685.279 phrases/sec :: (5:44:5) hours left\n",
      "(2:16:4) step 2940/15121, epoch 0 Training Loss = 0.16326 :: 685.278 phrases/sec :: (5:43:37) hours left\n",
      "(2:16:32) step 2950/15121, epoch 0 Training Loss = 0.16359 :: 685.354 phrases/sec :: (5:43:6) hours left\n",
      "(2:17:0) step 2960/15121, epoch 0 Training Loss = 0.16350 :: 685.392 phrases/sec :: (5:42:37) hours left\n",
      "(2:17:27) step 2970/15121, epoch 0 Training Loss = 0.16312 :: 685.499 phrases/sec :: (5:42:5) hours left\n",
      "(2:17:54) step 2980/15121, epoch 0 Training Loss = 0.16371 :: 685.662 phrases/sec :: (5:41:31) hours left\n",
      "(2:18:22) step 2990/15121, epoch 0 Training Loss = 0.16346 :: 685.638 phrases/sec :: (5:41:4) hours left\n",
      "(2:18:50) step 3000/15121, epoch 0 Training Loss = 0.16354 :: 685.627 phrases/sec :: (5:40:36) hours left\n",
      "(2:19:18) step 3010/15121, epoch 0 Training Loss = 0.16445 :: 685.700 phrases/sec :: (5:40:6) hours left\n",
      "(2:19:45) step 3020/15121, epoch 0 Training Loss = 0.16345 :: 685.730 phrases/sec :: (5:39:37) hours left\n",
      "(2:20:13) step 3030/15121, epoch 0 Training Loss = 0.16410 :: 685.811 phrases/sec :: (5:39:6) hours left\n",
      "(2:20:40) step 3040/15121, epoch 0 Training Loss = 0.16500 :: 685.951 phrases/sec :: (5:38:33) hours left\n",
      "(2:21:8) step 3050/15121, epoch 0 Training Loss = 0.16568 :: 685.929 phrases/sec :: (5:38:6) hours left\n",
      "(2:21:35) step 3060/15121, epoch 0 Training Loss = 0.16402 :: 686.069 phrases/sec :: (5:37:33) hours left\n",
      "(2:22:3) step 3070/15121, epoch 0 Training Loss = 0.16336 :: 686.057 phrases/sec :: (5:37:6) hours left\n",
      "(2:22:30) step 3080/15121, epoch 0 Training Loss = 0.16315 :: 686.138 phrases/sec :: (5:36:35) hours left\n",
      "(2:22:58) step 3090/15121, epoch 0 Training Loss = 0.16295 :: 686.134 phrases/sec :: (5:36:7) hours left\n",
      "(2:23:26) step 3100/15121, epoch 0 Training Loss = 0.16332 :: 686.142 phrases/sec :: (5:35:39) hours left\n",
      "(2:23:54) step 3110/15121, epoch 0 Training Loss = 0.16557 :: 686.141 phrases/sec :: (5:35:11) hours left\n",
      "(2:24:24) step 3120/15121, epoch 0 Training Loss = 0.16421 :: 685.903 phrases/sec :: (5:34:51) hours left\n",
      "(2:24:52) step 3130/15121, epoch 0 Training Loss = 0.16328 :: 685.927 phrases/sec :: (5:34:22) hours left\n",
      "(2:25:20) step 3140/15121, epoch 0 Training Loss = 0.16491 :: 685.864 phrases/sec :: (5:33:56) hours left\n",
      "(2:25:48) step 3150/15121, epoch 0 Training Loss = 0.16342 :: 685.859 phrases/sec :: (5:33:28) hours left\n",
      "(2:26:16) step 3160/15121, epoch 0 Training Loss = 0.16337 :: 685.936 phrases/sec :: (5:32:58) hours left\n",
      "(2:26:43) step 3170/15121, epoch 0 Training Loss = 0.16371 :: 685.975 phrases/sec :: (5:32:28) hours left\n",
      "(2:27:11) step 3180/15121, epoch 0 Training Loss = 0.16301 :: 685.987 phrases/sec :: (5:32:0) hours left\n",
      "(2:27:40) step 3190/15121, epoch 0 Training Loss = 0.16361 :: 685.913 phrases/sec :: (5:31:34) hours left\n",
      "(2:28:7) step 3200/15121, epoch 0 Training Loss = 0.16440 :: 686.077 phrases/sec :: (5:31:1) hours left\n",
      "(2:28:34) step 3210/15121, epoch 0 Training Loss = 0.16345 :: 686.177 phrases/sec :: (5:30:29) hours left\n",
      "(2:29:1) step 3220/15121, epoch 0 Training Loss = 0.16361 :: 686.261 phrases/sec :: (5:29:59) hours left\n",
      "(2:29:29) step 3230/15121, epoch 0 Training Loss = 0.16335 :: 686.288 phrases/sec :: (5:29:30) hours left\n",
      "(2:29:56) step 3240/15121, epoch 0 Training Loss = 0.16367 :: 686.336 phrases/sec :: (5:29:0) hours left\n",
      "Saving model...\n",
      "(2:30:26) step 3250/15121, epoch 0 Training Loss = 0.16360 :: 686.401 phrases/sec :: (5:28:28) hours left\n",
      "(2:30:54) step 3260/15121, epoch 0 Training Loss = 0.16392 :: 686.436 phrases/sec :: (5:27:59) hours left\n",
      "(2:31:21) step 3270/15121, epoch 0 Training Loss = 0.16362 :: 686.525 phrases/sec :: (5:27:28) hours left\n",
      "(2:31:48) step 3280/15121, epoch 0 Training Loss = 0.16502 :: 686.669 phrases/sec :: (5:26:55) hours left\n",
      "(2:32:14) step 3290/15121, epoch 0 Training Loss = 0.16309 :: 686.845 phrases/sec :: (5:26:21) hours left\n",
      "(2:32:41) step 3300/15121, epoch 0 Training Loss = 0.16339 :: 686.951 phrases/sec :: (5:25:50) hours left\n",
      "(2:33:8) step 3310/15121, epoch 0 Training Loss = 0.16369 :: 687.112 phrases/sec :: (5:25:16) hours left\n",
      "(2:33:35) step 3320/15121, epoch 0 Training Loss = 0.16352 :: 687.182 phrases/sec :: (5:24:46) hours left\n",
      "(2:34:4) step 3330/15121, epoch 0 Training Loss = 0.16378 :: 687.116 phrases/sec :: (5:24:20) hours left\n",
      "(2:34:32) step 3340/15121, epoch 0 Training Loss = 0.16583 :: 687.122 phrases/sec :: (5:23:52) hours left\n",
      "(2:35:0) step 3350/15121, epoch 0 Training Loss = 0.16386 :: 687.119 phrases/sec :: (5:23:24) hours left\n",
      "(2:35:27) step 3360/15121, epoch 0 Training Loss = 0.16328 :: 687.222 phrases/sec :: (5:22:53) hours left\n",
      "(2:35:55) step 3370/15121, epoch 0 Training Loss = 0.16414 :: 687.229 phrases/sec :: (5:22:24) hours left\n",
      "(2:36:22) step 3380/15121, epoch 0 Training Loss = 0.16396 :: 687.318 phrases/sec :: (5:21:53) hours left\n",
      "(2:36:50) step 3390/15121, epoch 0 Training Loss = 0.16367 :: 687.402 phrases/sec :: (5:21:23) hours left\n",
      "(2:37:18) step 3400/15121, epoch 0 Training Loss = 0.16328 :: 687.399 phrases/sec :: (5:20:55) hours left\n",
      "(2:37:46) step 3410/15121, epoch 0 Training Loss = 0.16488 :: 687.403 phrases/sec :: (5:20:26) hours left\n",
      "(2:38:15) step 3420/15121, epoch 0 Training Loss = 0.16346 :: 687.264 phrases/sec :: (5:20:3) hours left\n",
      "(2:38:44) step 3430/15121, epoch 0 Training Loss = 0.16355 :: 687.180 phrases/sec :: (5:19:37) hours left\n",
      "(2:39:12) step 3440/15121, epoch 0 Training Loss = 0.16620 :: 687.212 phrases/sec :: (5:19:8) hours left\n",
      "(2:39:39) step 3450/15121, epoch 0 Training Loss = 0.16397 :: 687.324 phrases/sec :: (5:18:37) hours left\n",
      "(2:40:7) step 3460/15121, epoch 0 Training Loss = 0.16336 :: 687.357 phrases/sec :: (5:18:8) hours left\n",
      "(2:40:34) step 3470/15121, epoch 0 Training Loss = 0.16404 :: 687.371 phrases/sec :: (5:17:39) hours left\n",
      "(2:41:2) step 3480/15121, epoch 0 Training Loss = 0.16322 :: 687.405 phrases/sec :: (5:17:10) hours left\n",
      "(2:41:30) step 3490/15121, epoch 0 Training Loss = 0.16338 :: 687.457 phrases/sec :: (5:16:40) hours left\n",
      "(2:41:56) step 3500/15121, epoch 0 Training Loss = 0.16392 :: 687.646 phrases/sec :: (5:16:6) hours left\n",
      "(2:42:24) step 3510/15121, epoch 0 Training Loss = 0.16330 :: 687.666 phrases/sec :: (5:15:38) hours left\n",
      "(2:42:52) step 3520/15121, epoch 0 Training Loss = 0.16366 :: 687.627 phrases/sec :: (5:15:11) hours left\n",
      "(2:43:22) step 3530/15121, epoch 0 Training Loss = 0.16358 :: 687.465 phrases/sec :: (5:14:48) hours left\n",
      "(2:44:0) step 3540/15121, epoch 0 Training Loss = 0.16419 :: 686.672 phrases/sec :: (5:14:42) hours left\n",
      "(2:44:37) step 3550/15121, epoch 0 Training Loss = 0.16423 :: 686.267 phrases/sec :: (5:14:22) hours left\n",
      "(2:45:10) step 3560/15121, epoch 0 Training Loss = 0.16395 :: 685.967 phrases/sec :: (5:14:2) hours left\n",
      "(2:45:39) step 3570/15121, epoch 0 Training Loss = 0.16351 :: 685.845 phrases/sec :: (5:13:38) hours left\n",
      "(2:46:7) step 3580/15121, epoch 0 Training Loss = 0.16452 :: 685.907 phrases/sec :: (5:13:8) hours left\n",
      "(2:46:35) step 3590/15121, epoch 0 Training Loss = 0.16332 :: 685.893 phrases/sec :: (5:12:40) hours left\n",
      "(2:47:2) step 3600/15121, epoch 0 Training Loss = 0.16389 :: 686.003 phrases/sec :: (5:12:8) hours left\n",
      "(2:47:29) step 3610/15121, epoch 0 Training Loss = 0.16352 :: 686.081 phrases/sec :: (5:11:38) hours left\n",
      "(2:47:56) step 3620/15121, epoch 0 Training Loss = 0.16313 :: 686.187 phrases/sec :: (5:11:7) hours left\n",
      "(2:48:24) step 3630/15121, epoch 0 Training Loss = 0.16459 :: 686.165 phrases/sec :: (5:10:39) hours left\n",
      "(2:48:56) step 3640/15121, epoch 0 Training Loss = 0.16490 :: 685.917 phrases/sec :: (5:10:18) hours left\n",
      "(2:49:25) step 3650/15121, epoch 0 Training Loss = 0.16687 :: 685.881 phrases/sec :: (5:9:51) hours left\n",
      "(2:49:52) step 3660/15121, epoch 0 Training Loss = 0.16348 :: 685.979 phrases/sec :: (5:9:20) hours left\n",
      "(2:50:18) step 3670/15121, epoch 0 Training Loss = 0.16341 :: 686.087 phrases/sec :: (5:8:49) hours left\n",
      "(2:50:46) step 3680/15121, epoch 0 Training Loss = 0.16366 :: 686.114 phrases/sec :: (5:8:20) hours left\n",
      "(2:51:14) step 3690/15121, epoch 0 Training Loss = 0.16579 :: 686.100 phrases/sec :: (5:7:52) hours left\n",
      "(2:51:41) step 3700/15121, epoch 0 Training Loss = 0.16340 :: 686.221 phrases/sec :: (5:7:20) hours left\n",
      "(2:52:10) step 3710/15121, epoch 0 Training Loss = 0.16466 :: 686.111 phrases/sec :: (5:6:56) hours left\n",
      "(2:52:38) step 3720/15121, epoch 0 Training Loss = 0.16323 :: 686.096 phrases/sec :: (5:6:29) hours left\n",
      "(2:53:5) step 3730/15121, epoch 0 Training Loss = 0.16338 :: 686.151 phrases/sec :: (5:5:59) hours left\n",
      "(2:53:34) step 3740/15121, epoch 0 Training Loss = 0.16333 :: 686.092 phrases/sec :: (5:5:33) hours left\n",
      "(2:54:1) step 3750/15121, epoch 0 Training Loss = 0.16357 :: 686.217 phrases/sec :: (5:5:1) hours left\n",
      "(2:54:27) step 3760/15121, epoch 0 Training Loss = 0.16320 :: 686.339 phrases/sec :: (5:4:29) hours left\n",
      "(2:54:55) step 3770/15121, epoch 0 Training Loss = 0.16375 :: 686.381 phrases/sec :: (5:4:0) hours left\n",
      "(2:55:24) step 3780/15121, epoch 0 Training Loss = 0.16393 :: 686.296 phrases/sec :: (5:3:35) hours left\n",
      "(2:55:51) step 3790/15121, epoch 0 Training Loss = 0.16319 :: 686.377 phrases/sec :: (5:3:4) hours left\n",
      "(2:56:20) step 3800/15121, epoch 0 Training Loss = 0.16360 :: 686.328 phrases/sec :: (5:2:37) hours left\n",
      "(2:56:47) step 3810/15121, epoch 0 Training Loss = 0.16540 :: 686.348 phrases/sec :: (5:2:9) hours left\n",
      "(2:57:15) step 3820/15121, epoch 0 Training Loss = 0.16358 :: 686.362 phrases/sec :: (5:1:40) hours left\n",
      "(2:57:44) step 3830/15121, epoch 0 Training Loss = 0.16513 :: 686.355 phrases/sec :: (5:1:12) hours left\n",
      "(2:58:12) step 3840/15121, epoch 0 Training Loss = 0.16475 :: 686.302 phrases/sec :: (5:0:46) hours left\n",
      "(2:58:40) step 3850/15121, epoch 0 Training Loss = 0.16333 :: 686.303 phrases/sec :: (5:0:18) hours left\n",
      "(2:59:8) step 3860/15121, epoch 0 Training Loss = 0.16308 :: 686.302 phrases/sec :: (4:59:50) hours left\n",
      "(2:59:36) step 3870/15121, epoch 0 Training Loss = 0.16337 :: 686.286 phrases/sec :: (4:59:22) hours left\n",
      "(3:0:6) step 3880/15121, epoch 0 Training Loss = 0.16603 :: 686.177 phrases/sec :: (4:58:58) hours left\n",
      "Saving model...\n",
      "(3:0:37) step 3890/15121, epoch 0 Training Loss = 0.16322 :: 686.244 phrases/sec :: (4:58:24) hours left\n",
      "(3:1:2) step 3900/15121, epoch 0 Training Loss = 0.16395 :: 686.224 phrases/sec :: (4:57:59) hours left\n",
      "(3:1:26) step 3910/15121, epoch 0 Training Loss = 0.16423 :: 686.393 phrases/sec :: (4:57:29) hours left\n",
      "(3:1:51) step 3920/15121, epoch 0 Training Loss = 0.16340 :: 686.459 phrases/sec :: (4:57:1) hours left\n",
      "(3:2:18) step 3930/15121, epoch 0 Training Loss = 0.16313 :: 686.394 phrases/sec :: (4:56:36) hours left\n",
      "(3:2:43) step 3940/15121, epoch 0 Training Loss = 0.16462 :: 686.500 phrases/sec :: (4:56:6) hours left\n",
      "(3:3:11) step 3950/15121, epoch 0 Training Loss = 0.16351 :: 686.458 phrases/sec :: (4:55:41) hours left\n",
      "(3:3:37) step 3960/15121, epoch 0 Training Loss = 0.16301 :: 686.506 phrases/sec :: (4:55:13) hours left\n",
      "(3:4:2) step 3970/15121, epoch 0 Training Loss = 0.16310 :: 686.634 phrases/sec :: (4:54:42) hours left\n",
      "(3:4:31) step 3980/15121, epoch 0 Training Loss = 0.16537 :: 686.471 phrases/sec :: (4:54:20) hours left\n",
      "(3:4:59) step 3990/15121, epoch 0 Training Loss = 0.16361 :: 686.423 phrases/sec :: (4:53:54) hours left\n",
      "(3:5:27) step 4000/15121, epoch 0 Training Loss = 0.16297 :: 686.339 phrases/sec :: (4:53:29) hours left\n",
      "(3:5:55) step 4010/15121, epoch 0 Training Loss = 0.16340 :: 686.303 phrases/sec :: (4:53:3) hours left\n",
      "(3:6:22) step 4020/15121, epoch 0 Training Loss = 0.16369 :: 686.291 phrases/sec :: (4:52:36) hours left\n",
      "(3:6:49) step 4030/15121, epoch 0 Training Loss = 0.16361 :: 686.340 phrases/sec :: (4:52:7) hours left\n",
      "(3:7:17) step 4040/15121, epoch 0 Training Loss = 0.16336 :: 686.291 phrases/sec :: (4:51:41) hours left\n",
      "(3:7:44) step 4050/15121, epoch 0 Training Loss = 0.16377 :: 686.334 phrases/sec :: (4:51:12) hours left\n",
      "(3:8:11) step 4060/15121, epoch 0 Training Loss = 0.16326 :: 686.422 phrases/sec :: (4:50:42) hours left\n",
      "(3:8:39) step 4070/15121, epoch 0 Training Loss = 0.16720 :: 686.405 phrases/sec :: (4:50:15) hours left\n",
      "(3:9:6) step 4080/15121, epoch 0 Training Loss = 0.16349 :: 686.421 phrases/sec :: (4:49:47) hours left\n",
      "(3:9:34) step 4090/15121, epoch 0 Training Loss = 0.16350 :: 686.368 phrases/sec :: (4:49:21) hours left\n",
      "(3:10:3) step 4100/15121, epoch 0 Training Loss = 0.16602 :: 686.324 phrases/sec :: (4:48:55) hours left\n",
      "(3:10:30) step 4110/15121, epoch 0 Training Loss = 0.16403 :: 686.380 phrases/sec :: (4:48:25) hours left\n",
      "(3:10:56) step 4120/15121, epoch 0 Training Loss = 0.16305 :: 686.479 phrases/sec :: (4:47:54) hours left\n",
      "(3:11:24) step 4130/15121, epoch 0 Training Loss = 0.16335 :: 686.477 phrases/sec :: (4:47:27) hours left\n",
      "(3:11:52) step 4140/15121, epoch 0 Training Loss = 0.16309 :: 686.472 phrases/sec :: (4:46:59) hours left\n",
      "(3:12:22) step 4150/15121, epoch 0 Training Loss = 0.16356 :: 686.282 phrases/sec :: (4:46:37) hours left\n",
      "(3:12:50) step 4160/15121, epoch 0 Training Loss = 0.16389 :: 686.222 phrases/sec :: (4:46:11) hours left\n",
      "(3:13:18) step 4170/15121, epoch 0 Training Loss = 0.16376 :: 686.184 phrases/sec :: (4:45:44) hours left\n",
      "(3:13:48) step 4180/15121, epoch 0 Training Loss = 0.16358 :: 686.030 phrases/sec :: (4:45:21) hours left\n",
      "(3:14:16) step 4190/15121, epoch 0 Training Loss = 0.16332 :: 685.992 phrases/sec :: (4:44:55) hours left\n",
      "(3:14:45) step 4200/15121, epoch 0 Training Loss = 0.16331 :: 685.927 phrases/sec :: (4:44:29) hours left\n",
      "(3:15:13) step 4210/15121, epoch 0 Training Loss = 0.16502 :: 685.903 phrases/sec :: (4:44:2) hours left\n",
      "(3:15:42) step 4220/15121, epoch 0 Training Loss = 0.16353 :: 685.808 phrases/sec :: (4:43:37) hours left\n",
      "(3:16:10) step 4230/15121, epoch 0 Training Loss = 0.16396 :: 685.822 phrases/sec :: (4:43:9) hours left\n",
      "(3:16:37) step 4240/15121, epoch 0 Training Loss = 0.16347 :: 685.839 phrases/sec :: (4:42:40) hours left\n",
      "(3:17:6) step 4250/15121, epoch 0 Training Loss = 0.16319 :: 685.816 phrases/sec :: (4:42:13) hours left\n",
      "(3:17:37) step 4260/15121, epoch 0 Training Loss = 0.16315 :: 685.625 phrases/sec :: (4:41:50) hours left\n",
      "(3:18:5) step 4270/15121, epoch 0 Training Loss = 0.16409 :: 685.619 phrases/sec :: (4:41:22) hours left\n",
      "(3:18:31) step 4280/15121, epoch 0 Training Loss = 0.16304 :: 685.726 phrases/sec :: (4:40:51) hours left\n",
      "(3:19:0) step 4290/15121, epoch 0 Training Loss = 0.16351 :: 685.653 phrases/sec :: (4:40:25) hours left\n",
      "(3:19:28) step 4300/15121, epoch 0 Training Loss = 0.16354 :: 685.642 phrases/sec :: (4:39:57) hours left\n",
      "(3:19:56) step 4310/15121, epoch 0 Training Loss = 0.16321 :: 685.634 phrases/sec :: (4:39:30) hours left\n",
      "(3:20:24) step 4320/15121, epoch 0 Training Loss = 0.16456 :: 685.655 phrases/sec :: (4:39:1) hours left\n",
      "(3:20:51) step 4330/15121, epoch 0 Training Loss = 0.16315 :: 685.707 phrases/sec :: (4:38:32) hours left\n",
      "(3:21:18) step 4340/15121, epoch 0 Training Loss = 0.16344 :: 685.772 phrases/sec :: (4:38:2) hours left\n",
      "(3:21:46) step 4350/15121, epoch 0 Training Loss = 0.16405 :: 685.773 phrases/sec :: (4:37:34) hours left\n",
      "(3:22:14) step 4360/15121, epoch 0 Training Loss = 0.16320 :: 685.806 phrases/sec :: (4:37:5) hours left\n",
      "(3:22:43) step 4370/15121, epoch 0 Training Loss = 0.16316 :: 685.717 phrases/sec :: (4:36:40) hours left\n",
      "(3:23:10) step 4380/15121, epoch 0 Training Loss = 0.16358 :: 685.794 phrases/sec :: (4:36:9) hours left\n",
      "(3:23:38) step 4390/15121, epoch 0 Training Loss = 0.16346 :: 685.784 phrases/sec :: (4:35:42) hours left\n",
      "(3:24:5) step 4400/15121, epoch 0 Training Loss = 0.16462 :: 685.829 phrases/sec :: (4:35:12) hours left\n",
      "(3:24:35) step 4410/15121, epoch 0 Training Loss = 0.16419 :: 685.778 phrases/sec :: (4:34:45) hours left\n",
      "(3:25:3) step 4420/15121, epoch 0 Training Loss = 0.16426 :: 685.799 phrases/sec :: (4:34:16) hours left\n",
      "(3:25:31) step 4430/15121, epoch 0 Training Loss = 0.16336 :: 685.779 phrases/sec :: (4:33:49) hours left\n",
      "(3:26:0) step 4440/15121, epoch 0 Training Loss = 0.16335 :: 685.722 phrases/sec :: (4:33:22) hours left\n",
      "(3:26:28) step 4450/15121, epoch 0 Training Loss = 0.16340 :: 685.707 phrases/sec :: (4:32:55) hours left\n",
      "(3:26:55) step 4460/15121, epoch 0 Training Loss = 0.16341 :: 685.798 phrases/sec :: (4:32:24) hours left\n",
      "(3:27:23) step 4470/15121, epoch 0 Training Loss = 0.16453 :: 685.836 phrases/sec :: (4:31:55) hours left\n",
      "(3:27:50) step 4480/15121, epoch 0 Training Loss = 0.16365 :: 685.887 phrases/sec :: (4:31:25) hours left\n",
      "(3:28:17) step 4490/15121, epoch 0 Training Loss = 0.16327 :: 685.960 phrases/sec :: (4:30:55) hours left\n",
      "(3:28:44) step 4500/15121, epoch 0 Training Loss = 0.16339 :: 686.042 phrases/sec :: (4:30:25) hours left\n",
      "(3:29:12) step 4510/15121, epoch 0 Training Loss = 0.16310 :: 686.049 phrases/sec :: (4:29:57) hours left\n",
      "(3:29:41) step 4520/15121, epoch 0 Training Loss = 0.16339 :: 685.930 phrases/sec :: (4:29:32) hours left\n",
      "(3:30:9) step 4530/15121, epoch 0 Training Loss = 0.16336 :: 685.943 phrases/sec :: (4:29:4) hours left\n",
      "Saving model...\n",
      "(3:30:39) step 4540/15121, epoch 0 Training Loss = 0.16399 :: 685.953 phrases/sec :: (4:28:33) hours left\n",
      "(3:31:9) step 4550/15121, epoch 0 Training Loss = 0.16466 :: 685.775 phrases/sec :: (4:28:11) hours left\n",
      "(3:31:38) step 4560/15121, epoch 0 Training Loss = 0.16329 :: 685.757 phrases/sec :: (4:27:43) hours left\n",
      "(3:32:6) step 4570/15121, epoch 0 Training Loss = 0.16330 :: 685.708 phrases/sec :: (4:27:17) hours left\n",
      "(3:32:34) step 4580/15121, epoch 0 Training Loss = 0.16353 :: 685.753 phrases/sec :: (4:26:47) hours left\n",
      "(3:33:2) step 4590/15121, epoch 0 Training Loss = 0.16319 :: 685.720 phrases/sec :: (4:26:20) hours left\n",
      "(3:33:29) step 4600/15121, epoch 0 Training Loss = 0.16328 :: 685.810 phrases/sec :: (4:25:50) hours left\n",
      "(3:33:56) step 4610/15121, epoch 0 Training Loss = 0.16355 :: 685.851 phrases/sec :: (4:25:21) hours left\n",
      "(3:34:25) step 4620/15121, epoch 0 Training Loss = 0.16346 :: 685.808 phrases/sec :: (4:24:54) hours left\n",
      "(3:34:52) step 4630/15121, epoch 0 Training Loss = 0.16313 :: 685.836 phrases/sec :: (4:24:26) hours left\n",
      "(3:35:20) step 4640/15121, epoch 0 Training Loss = 0.16381 :: 685.803 phrases/sec :: (4:23:59) hours left\n",
      "(3:35:48) step 4650/15121, epoch 0 Training Loss = 0.16316 :: 685.785 phrases/sec :: (4:23:31) hours left\n",
      "(3:36:16) step 4660/15121, epoch 0 Training Loss = 0.16304 :: 685.835 phrases/sec :: (4:23:2) hours left\n",
      "(3:36:42) step 4670/15121, epoch 0 Training Loss = 0.16596 :: 685.981 phrases/sec :: (4:22:30) hours left\n",
      "(3:37:11) step 4680/15121, epoch 0 Training Loss = 0.16400 :: 685.897 phrases/sec :: (4:22:4) hours left\n",
      "(3:37:39) step 4690/15121, epoch 0 Training Loss = 0.16387 :: 685.886 phrases/sec :: (4:21:36) hours left\n",
      "(3:38:7) step 4700/15121, epoch 0 Training Loss = 0.16364 :: 685.887 phrases/sec :: (4:21:8) hours left\n",
      "(3:38:34) step 4710/15121, epoch 0 Training Loss = 0.16300 :: 685.949 phrases/sec :: (4:20:38) hours left\n",
      "(3:39:2) step 4720/15121, epoch 0 Training Loss = 0.16326 :: 685.973 phrases/sec :: (4:20:10) hours left\n",
      "(3:39:30) step 4730/15121, epoch 0 Training Loss = 0.16540 :: 685.976 phrases/sec :: (4:19:42) hours left\n",
      "(3:39:58) step 4740/15121, epoch 0 Training Loss = 0.16307 :: 685.985 phrases/sec :: (4:19:13) hours left\n",
      "(3:40:24) step 4750/15121, epoch 0 Training Loss = 0.16304 :: 686.109 phrases/sec :: (4:18:42) hours left\n",
      "(3:40:54) step 4760/15121, epoch 0 Training Loss = 0.16406 :: 685.989 phrases/sec :: (4:18:17) hours left\n",
      "(3:41:21) step 4770/15121, epoch 0 Training Loss = 0.16347 :: 686.077 phrases/sec :: (4:17:47) hours left\n",
      "(3:41:50) step 4780/15121, epoch 0 Training Loss = 0.16341 :: 685.944 phrases/sec :: (4:17:23) hours left\n",
      "(3:42:18) step 4790/15121, epoch 0 Training Loss = 0.16358 :: 685.985 phrases/sec :: (4:16:54) hours left\n",
      "(3:42:44) step 4800/15121, epoch 0 Training Loss = 0.16337 :: 686.073 phrases/sec :: (4:16:23) hours left\n",
      "(3:43:13) step 4810/15121, epoch 0 Training Loss = 0.16349 :: 686.028 phrases/sec :: (4:15:57) hours left\n",
      "(3:43:40) step 4820/15121, epoch 0 Training Loss = 0.16349 :: 686.057 phrases/sec :: (4:15:28) hours left\n",
      "(3:44:8) step 4830/15121, epoch 0 Training Loss = 0.16317 :: 686.058 phrases/sec :: (4:15:0) hours left\n",
      "(3:44:37) step 4840/15121, epoch 0 Training Loss = 0.16362 :: 685.995 phrases/sec :: (4:14:34) hours left\n",
      "(3:45:4) step 4850/15121, epoch 0 Training Loss = 0.16431 :: 686.077 phrases/sec :: (4:14:4) hours left\n",
      "(3:45:32) step 4860/15121, epoch 0 Training Loss = 0.16328 :: 686.071 phrases/sec :: (4:13:36) hours left\n",
      "(3:46:0) step 4870/15121, epoch 0 Training Loss = 0.16363 :: 686.046 phrases/sec :: (4:13:9) hours left\n",
      "(3:46:30) step 4880/15121, epoch 0 Training Loss = 0.16400 :: 685.934 phrases/sec :: (4:12:44) hours left\n",
      "(3:46:56) step 4890/15121, epoch 0 Training Loss = 0.16314 :: 686.027 phrases/sec :: (4:12:13) hours left\n",
      "(3:47:23) step 4900/15121, epoch 0 Training Loss = 0.16319 :: 686.094 phrases/sec :: (4:11:43) hours left\n",
      "(3:47:51) step 4910/15121, epoch 0 Training Loss = 0.16321 :: 686.120 phrases/sec :: (4:11:14) hours left\n",
      "(3:48:20) step 4920/15121, epoch 0 Training Loss = 0.16349 :: 686.029 phrases/sec :: (4:10:49) hours left\n",
      "(3:48:47) step 4930/15121, epoch 0 Training Loss = 0.16369 :: 686.114 phrases/sec :: (4:10:19) hours left\n",
      "(3:49:15) step 4940/15121, epoch 0 Training Loss = 0.16321 :: 686.134 phrases/sec :: (4:9:50) hours left\n",
      "(3:49:42) step 4950/15121, epoch 0 Training Loss = 0.16375 :: 686.166 phrases/sec :: (4:9:21) hours left\n",
      "(3:50:10) step 4960/15121, epoch 0 Training Loss = 0.16347 :: 686.209 phrases/sec :: (4:8:52) hours left\n",
      "(3:50:37) step 4970/15121, epoch 0 Training Loss = 0.16301 :: 686.266 phrases/sec :: (4:8:23) hours left\n",
      "(3:51:4) step 4980/15121, epoch 0 Training Loss = 0.16339 :: 686.307 phrases/sec :: (4:7:54) hours left\n",
      "(3:51:33) step 4990/15121, epoch 0 Training Loss = 0.16339 :: 686.249 phrases/sec :: (4:7:27) hours left\n",
      "(3:52:0) step 5000/15121, epoch 0 Training Loss = 0.16372 :: 686.276 phrases/sec :: (4:6:59) hours left\n",
      "(3:52:28) step 5010/15121, epoch 0 Training Loss = 0.16354 :: 686.309 phrases/sec :: (4:6:30) hours left\n",
      "(3:52:55) step 5020/15121, epoch 0 Training Loss = 0.16319 :: 686.371 phrases/sec :: (4:6:0) hours left\n",
      "(3:53:24) step 5030/15121, epoch 0 Training Loss = 0.16351 :: 686.320 phrases/sec :: (4:5:34) hours left\n",
      "(3:53:53) step 5040/15121, epoch 0 Training Loss = 0.16329 :: 686.250 phrases/sec :: (4:5:8) hours left\n",
      "(3:54:22) step 5050/15121, epoch 0 Training Loss = 0.16363 :: 686.166 phrases/sec :: (4:4:42) hours left\n",
      "(3:54:50) step 5060/15121, epoch 0 Training Loss = 0.16345 :: 686.119 phrases/sec :: (4:4:15) hours left\n",
      "(3:55:19) step 5070/15121, epoch 0 Training Loss = 0.16351 :: 686.103 phrases/sec :: (4:3:48) hours left\n",
      "(3:55:46) step 5080/15121, epoch 0 Training Loss = 0.16497 :: 686.167 phrases/sec :: (4:3:18) hours left\n",
      "(3:56:14) step 5090/15121, epoch 0 Training Loss = 0.16354 :: 686.121 phrases/sec :: (4:2:51) hours left\n",
      "(3:56:43) step 5100/15121, epoch 0 Training Loss = 0.16637 :: 686.064 phrases/sec :: (4:2:25) hours left\n",
      "(3:57:12) step 5110/15121, epoch 0 Training Loss = 0.16367 :: 685.988 phrases/sec :: (4:1:59) hours left\n",
      "(3:57:41) step 5120/15121, epoch 0 Training Loss = 0.16378 :: 685.943 phrases/sec :: (4:1:32) hours left\n",
      "(3:58:8) step 5130/15121, epoch 0 Training Loss = 0.16342 :: 685.980 phrases/sec :: (4:1:3) hours left\n",
      "(3:58:38) step 5140/15121, epoch 0 Training Loss = 0.16330 :: 685.869 phrases/sec :: (4:0:39) hours left\n",
      "(3:59:4) step 5150/15121, epoch 0 Training Loss = 0.16307 :: 685.932 phrases/sec :: (4:0:9) hours left\n",
      "(3:59:33) step 5160/15121, epoch 0 Training Loss = 0.16383 :: 685.899 phrases/sec :: (3:59:42) hours left\n",
      "(4:0:0) step 5170/15121, epoch 0 Training Loss = 0.16354 :: 685.970 phrases/sec :: (3:59:12) hours left\n",
      "(4:0:28) step 5180/15121, epoch 0 Training Loss = 0.16363 :: 685.959 phrases/sec :: (3:58:44) hours left\n",
      "Saving model...\n",
      "(4:1:4) step 5190/15121, epoch 0 Training Loss = 0.16370 :: 685.715 phrases/sec :: (3:58:19) hours left\n",
      "(4:1:46) step 5200/15121, epoch 0 Training Loss = 0.16418 :: 685.133 phrases/sec :: (3:58:1) hours left\n",
      "(4:2:21) step 5210/15121, epoch 0 Training Loss = 0.16361 :: 684.959 phrases/sec :: (3:57:33) hours left\n",
      "(4:2:50) step 5220/15121, epoch 0 Training Loss = 0.16332 :: 684.962 phrases/sec :: (3:57:4) hours left\n",
      "(4:3:18) step 5230/15121, epoch 0 Training Loss = 0.16326 :: 684.960 phrases/sec :: (3:56:36) hours left\n",
      "(4:3:45) step 5240/15121, epoch 0 Training Loss = 0.16360 :: 685.030 phrases/sec :: (3:56:6) hours left\n",
      "(4:4:12) step 5250/15121, epoch 0 Training Loss = 0.16709 :: 685.082 phrases/sec :: (3:55:37) hours left\n",
      "(4:4:40) step 5260/15121, epoch 0 Training Loss = 0.16392 :: 685.115 phrases/sec :: (3:55:8) hours left\n",
      "(4:5:8) step 5270/15121, epoch 0 Training Loss = 0.16335 :: 685.121 phrases/sec :: (3:54:40) hours left\n",
      "(4:5:36) step 5280/15121, epoch 0 Training Loss = 0.16352 :: 685.087 phrases/sec :: (3:54:13) hours left\n",
      "(4:6:4) step 5290/15121, epoch 0 Training Loss = 0.16362 :: 685.078 phrases/sec :: (3:53:45) hours left\n",
      "(4:6:32) step 5300/15121, epoch 0 Training Loss = 0.16328 :: 685.100 phrases/sec :: (3:53:16) hours left\n",
      "(4:7:0) step 5310/15121, epoch 0 Training Loss = 0.16324 :: 685.117 phrases/sec :: (3:52:48) hours left\n",
      "(4:7:27) step 5320/15121, epoch 0 Training Loss = 0.16321 :: 685.219 phrases/sec :: (3:52:17) hours left\n",
      "(4:7:54) step 5330/15121, epoch 0 Training Loss = 0.16310 :: 685.241 phrases/sec :: (3:51:48) hours left\n",
      "(4:8:22) step 5340/15121, epoch 0 Training Loss = 0.16323 :: 685.241 phrases/sec :: (3:51:20) hours left\n",
      "(4:8:51) step 5350/15121, epoch 0 Training Loss = 0.16361 :: 685.225 phrases/sec :: (3:50:52) hours left\n",
      "(4:9:20) step 5360/15121, epoch 0 Training Loss = 0.16336 :: 685.143 phrases/sec :: (3:50:27) hours left\n",
      "(4:9:48) step 5370/15121, epoch 0 Training Loss = 0.16318 :: 685.116 phrases/sec :: (3:49:59) hours left\n",
      "(4:10:18) step 5380/15121, epoch 0 Training Loss = 0.16337 :: 685.032 phrases/sec :: (3:49:34) hours left\n",
      "(4:10:45) step 5390/15121, epoch 0 Training Loss = 0.16349 :: 685.069 phrases/sec :: (3:49:4) hours left\n",
      "(4:11:13) step 5400/15121, epoch 0 Training Loss = 0.16334 :: 685.061 phrases/sec :: (3:48:37) hours left\n",
      "(4:11:40) step 5410/15121, epoch 0 Training Loss = 0.16380 :: 685.181 phrases/sec :: (3:48:5) hours left\n",
      "(4:12:9) step 5420/15121, epoch 0 Training Loss = 0.16323 :: 685.087 phrases/sec :: (3:47:40) hours left\n",
      "(4:12:39) step 5430/15121, epoch 0 Training Loss = 0.16325 :: 685.027 phrases/sec :: (3:47:12) hours left\n",
      "(4:13:11) step 5440/15121, epoch 0 Training Loss = 0.16328 :: 684.932 phrases/sec :: (3:46:44) hours left\n",
      "(4:13:44) step 5450/15121, epoch 0 Training Loss = 0.16363 :: 684.811 phrases/sec :: (3:46:17) hours left\n",
      "(4:14:17) step 5460/15121, epoch 0 Training Loss = 0.16310 :: 684.536 phrases/sec :: (3:45:55) hours left\n",
      "(4:14:46) step 5470/15121, epoch 0 Training Loss = 0.16320 :: 684.532 phrases/sec :: (3:45:27) hours left\n",
      "(4:15:15) step 5480/15121, epoch 0 Training Loss = 0.16336 :: 684.462 phrases/sec :: (3:45:0) hours left\n",
      "(4:15:43) step 5490/15121, epoch 0 Training Loss = 0.16482 :: 684.436 phrases/sec :: (3:44:33) hours left\n",
      "(4:16:10) step 5500/15121, epoch 0 Training Loss = 0.16373 :: 684.497 phrases/sec :: (3:44:3) hours left\n",
      "(4:16:38) step 5510/15121, epoch 0 Training Loss = 0.16321 :: 684.540 phrases/sec :: (3:43:34) hours left\n",
      "(4:17:7) step 5520/15121, epoch 0 Training Loss = 0.16352 :: 684.497 phrases/sec :: (3:43:7) hours left\n",
      "(4:17:37) step 5530/15121, epoch 0 Training Loss = 0.16330 :: 684.371 phrases/sec :: (3:42:42) hours left\n",
      "(4:18:5) step 5540/15121, epoch 0 Training Loss = 0.16347 :: 684.329 phrases/sec :: (3:42:15) hours left\n",
      "(4:18:32) step 5550/15121, epoch 0 Training Loss = 0.17293 :: 684.430 phrases/sec :: (3:41:45) hours left\n",
      "(4:19:2) step 5560/15121, epoch 0 Training Loss = 0.16319 :: 684.405 phrases/sec :: (3:41:16) hours left\n",
      "(4:19:33) step 5570/15121, epoch 0 Training Loss = 0.16339 :: 684.326 phrases/sec :: (3:40:47) hours left\n",
      "(4:20:7) step 5580/15121, epoch 0 Training Loss = 0.16376 :: 684.130 phrases/sec :: (3:40:22) hours left\n",
      "(4:20:40) step 5590/15121, epoch 0 Training Loss = 0.16352 :: 683.978 phrases/sec :: (3:39:55) hours left\n",
      "(4:21:8) step 5600/15121, epoch 0 Training Loss = 0.16489 :: 684.034 phrases/sec :: (3:39:25) hours left\n",
      "(4:21:36) step 5610/15121, epoch 0 Training Loss = 0.16339 :: 683.995 phrases/sec :: (3:38:58) hours left\n",
      "(4:22:4) step 5620/15121, epoch 0 Training Loss = 0.16327 :: 684.044 phrases/sec :: (3:38:29) hours left\n",
      "(4:22:32) step 5630/15121, epoch 0 Training Loss = 0.16340 :: 684.053 phrases/sec :: (3:38:1) hours left\n",
      "(4:23:0) step 5640/15121, epoch 0 Training Loss = 0.16369 :: 684.026 phrases/sec :: (3:37:33) hours left\n",
      "(4:23:27) step 5650/15121, epoch 0 Training Loss = 0.16330 :: 684.101 phrases/sec :: (3:37:3) hours left\n",
      "(4:23:54) step 5660/15121, epoch 0 Training Loss = 0.16366 :: 684.159 phrases/sec :: (3:36:34) hours left\n",
      "(4:24:21) step 5670/15121, epoch 0 Training Loss = 0.16345 :: 684.232 phrases/sec :: (3:36:4) hours left\n",
      "(4:24:49) step 5680/15121, epoch 0 Training Loss = 0.16385 :: 684.266 phrases/sec :: (3:35:35) hours left\n",
      "(4:25:16) step 5690/15121, epoch 0 Training Loss = 0.16315 :: 684.336 phrases/sec :: (3:35:5) hours left\n",
      "(4:25:42) step 5700/15121, epoch 0 Training Loss = 0.16409 :: 684.423 phrases/sec :: (3:34:34) hours left\n",
      "(4:26:10) step 5710/15121, epoch 0 Training Loss = 0.16305 :: 684.428 phrases/sec :: (3:34:6) hours left\n",
      "(4:26:39) step 5720/15121, epoch 0 Training Loss = 0.16375 :: 684.382 phrases/sec :: (3:33:39) hours left\n",
      "(4:27:6) step 5730/15121, epoch 0 Training Loss = 0.16353 :: 684.470 phrases/sec :: (3:33:9) hours left\n",
      "(4:27:34) step 5740/15121, epoch 0 Training Loss = 0.16352 :: 684.460 phrases/sec :: (3:32:41) hours left\n",
      "(4:28:0) step 5750/15121, epoch 0 Training Loss = 0.16324 :: 684.531 phrases/sec :: (3:32:12) hours left\n",
      "(4:28:28) step 5760/15121, epoch 0 Training Loss = 0.16348 :: 684.569 phrases/sec :: (3:31:43) hours left\n",
      "(4:28:57) step 5770/15121, epoch 0 Training Loss = 0.16356 :: 684.507 phrases/sec :: (3:31:16) hours left\n",
      "(4:29:24) step 5780/15121, epoch 0 Training Loss = 0.16367 :: 684.529 phrases/sec :: (3:30:48) hours left\n",
      "(4:29:53) step 5790/15121, epoch 0 Training Loss = 0.16368 :: 684.489 phrases/sec :: (3:30:21) hours left\n",
      "(4:30:20) step 5800/15121, epoch 0 Training Loss = 0.16393 :: 684.531 phrases/sec :: (3:29:52) hours left\n",
      "Saving model...\n",
      "(4:30:50) step 5810/15121, epoch 0 Training Loss = 0.16355 :: 684.579 phrases/sec :: (3:29:20) hours left\n",
      "(4:31:16) step 5820/15121, epoch 0 Training Loss = 0.16337 :: 684.455 phrases/sec :: (3:29:0) hours left\n",
      "(4:31:42) step 5830/15121, epoch 0 Training Loss = 0.16319 :: 684.302 phrases/sec :: (3:28:40) hours left\n",
      "(4:32:5) step 5840/15121, epoch 0 Training Loss = 0.16397 :: 684.406 phrases/sec :: (3:28:13) hours left\n",
      "(4:32:30) step 5850/15121, epoch 0 Training Loss = 0.16300 :: 684.393 phrases/sec :: (3:27:48) hours left\n",
      "(4:32:56) step 5860/15121, epoch 0 Training Loss = 0.16308 :: 684.344 phrases/sec :: (3:27:24) hours left\n",
      "(4:33:21) step 5870/15121, epoch 0 Training Loss = 0.16412 :: 684.375 phrases/sec :: (3:26:58) hours left\n",
      "(4:33:49) step 5880/15121, epoch 0 Training Loss = 0.16660 :: 684.261 phrases/sec :: (3:26:35) hours left\n",
      "(4:34:14) step 5890/15121, epoch 0 Training Loss = 0.16330 :: 684.290 phrases/sec :: (3:26:8) hours left\n",
      "(4:34:40) step 5900/15121, epoch 0 Training Loss = 0.16332 :: 684.307 phrases/sec :: (3:25:42) hours left\n",
      "(4:35:7) step 5910/15121, epoch 0 Training Loss = 0.16519 :: 684.255 phrases/sec :: (3:25:16) hours left\n",
      "(4:35:32) step 5920/15121, epoch 0 Training Loss = 0.16329 :: 684.359 phrases/sec :: (3:24:47) hours left\n",
      "(4:35:59) step 5930/15121, epoch 0 Training Loss = 0.16324 :: 684.342 phrases/sec :: (3:24:21) hours left\n",
      "(4:36:25) step 5940/15121, epoch 0 Training Loss = 0.16316 :: 684.402 phrases/sec :: (3:23:53) hours left\n",
      "(4:36:50) step 5950/15121, epoch 0 Training Loss = 0.16344 :: 684.469 phrases/sec :: (3:23:24) hours left\n",
      "(4:37:17) step 5960/15121, epoch 0 Training Loss = 0.16525 :: 684.506 phrases/sec :: (3:22:56) hours left\n",
      "(4:37:43) step 5970/15121, epoch 0 Training Loss = 0.16376 :: 684.549 phrases/sec :: (3:22:28) hours left\n",
      "(4:38:10) step 5980/15121, epoch 0 Training Loss = 0.16394 :: 684.586 phrases/sec :: (3:22:0) hours left\n",
      "(4:38:36) step 5990/15121, epoch 0 Training Loss = 0.16381 :: 684.637 phrases/sec :: (3:21:32) hours left\n",
      "(4:39:3) step 6000/15121, epoch 0 Training Loss = 0.16335 :: 684.617 phrases/sec :: (3:21:5) hours left\n",
      "(4:39:31) step 6010/15121, epoch 0 Training Loss = 0.16321 :: 684.622 phrases/sec :: (3:20:38) hours left\n",
      "(4:39:58) step 6020/15121, epoch 0 Training Loss = 0.16328 :: 684.645 phrases/sec :: (3:20:10) hours left\n",
      "(4:40:24) step 6030/15121, epoch 0 Training Loss = 0.16447 :: 684.693 phrases/sec :: (3:19:41) hours left\n",
      "(4:40:52) step 6040/15121, epoch 0 Training Loss = 0.16378 :: 684.702 phrases/sec :: (3:19:13) hours left\n",
      "(4:41:18) step 6050/15121, epoch 0 Training Loss = 0.16531 :: 684.771 phrases/sec :: (3:18:44) hours left\n",
      "(4:41:46) step 6060/15121, epoch 0 Training Loss = 0.16317 :: 684.756 phrases/sec :: (3:18:17) hours left\n",
      "(4:42:13) step 6070/15121, epoch 0 Training Loss = 0.16335 :: 684.780 phrases/sec :: (3:17:48) hours left\n",
      "(4:42:40) step 6080/15121, epoch 0 Training Loss = 0.16335 :: 684.828 phrases/sec :: (3:17:20) hours left\n",
      "(4:43:7) step 6090/15121, epoch 0 Training Loss = 0.16335 :: 684.850 phrases/sec :: (3:16:51) hours left\n",
      "(4:43:35) step 6100/15121, epoch 0 Training Loss = 0.16312 :: 684.855 phrases/sec :: (3:16:23) hours left\n",
      "(4:44:4) step 6110/15121, epoch 0 Training Loss = 0.16299 :: 684.813 phrases/sec :: (3:15:57) hours left\n",
      "(4:44:31) step 6120/15121, epoch 0 Training Loss = 0.16362 :: 684.822 phrases/sec :: (3:15:29) hours left\n",
      "(4:45:0) step 6130/15121, epoch 0 Training Loss = 0.16318 :: 684.761 phrases/sec :: (3:15:3) hours left\n",
      "(4:45:29) step 6140/15121, epoch 0 Training Loss = 0.16316 :: 684.687 phrases/sec :: (3:14:37) hours left\n",
      "(4:45:56) step 6150/15121, epoch 0 Training Loss = 0.16340 :: 684.759 phrases/sec :: (3:14:7) hours left\n",
      "(4:46:24) step 6160/15121, epoch 0 Training Loss = 0.16323 :: 684.702 phrases/sec :: (3:13:41) hours left\n",
      "(4:46:53) step 6170/15121, epoch 0 Training Loss = 0.16410 :: 684.636 phrases/sec :: (3:13:14) hours left\n",
      "(4:47:21) step 6180/15121, epoch 0 Training Loss = 0.16286 :: 684.654 phrases/sec :: (3:12:46) hours left\n",
      "(4:47:48) step 6190/15121, epoch 0 Training Loss = 0.16318 :: 684.715 phrases/sec :: (3:12:17) hours left\n",
      "(4:48:14) step 6200/15121, epoch 0 Training Loss = 0.16332 :: 684.785 phrases/sec :: (3:11:47) hours left\n",
      "(4:48:42) step 6210/15121, epoch 0 Training Loss = 0.16332 :: 684.781 phrases/sec :: (3:11:19) hours left\n",
      "(4:49:9) step 6220/15121, epoch 0 Training Loss = 0.16354 :: 684.875 phrases/sec :: (3:10:49) hours left\n",
      "(4:49:35) step 6230/15121, epoch 0 Training Loss = 0.16332 :: 684.973 phrases/sec :: (3:10:19) hours left\n",
      "(4:50:2) step 6240/15121, epoch 0 Training Loss = 0.16353 :: 685.058 phrases/sec :: (3:9:49) hours left\n",
      "(4:50:32) step 6250/15121, epoch 0 Training Loss = 0.16447 :: 684.927 phrases/sec :: (3:9:24) hours left\n",
      "(4:51:0) step 6260/15121, epoch 0 Training Loss = 0.16574 :: 684.916 phrases/sec :: (3:8:56) hours left\n",
      "(4:51:26) step 6270/15121, epoch 0 Training Loss = 0.16383 :: 685.013 phrases/sec :: (3:8:26) hours left\n",
      "(4:51:53) step 6280/15121, epoch 0 Training Loss = 0.16550 :: 685.085 phrases/sec :: (3:7:56) hours left\n",
      "(4:52:21) step 6290/15121, epoch 0 Training Loss = 0.16337 :: 685.053 phrases/sec :: (3:7:29) hours left\n",
      "(4:52:49) step 6300/15121, epoch 0 Training Loss = 0.16374 :: 685.021 phrases/sec :: (3:7:2) hours left\n",
      "(4:53:19) step 6310/15121, epoch 0 Training Loss = 0.16509 :: 684.953 phrases/sec :: (3:6:36) hours left\n",
      "(4:53:46) step 6320/15121, epoch 0 Training Loss = 0.16502 :: 684.969 phrases/sec :: (3:6:8) hours left\n",
      "(4:54:14) step 6330/15121, epoch 0 Training Loss = 0.16445 :: 684.959 phrases/sec :: (3:5:40) hours left\n",
      "(4:54:42) step 6340/15121, epoch 0 Training Loss = 0.16344 :: 684.978 phrases/sec :: (3:5:12) hours left\n",
      "(4:55:9) step 6350/15121, epoch 0 Training Loss = 0.16331 :: 685.039 phrases/sec :: (3:4:42) hours left\n",
      "(4:55:36) step 6360/15121, epoch 0 Training Loss = 0.16312 :: 685.053 phrases/sec :: (3:4:14) hours left\n",
      "(4:56:3) step 6370/15121, epoch 0 Training Loss = 0.16323 :: 685.117 phrases/sec :: (3:3:45) hours left\n",
      "(4:56:31) step 6380/15121, epoch 0 Training Loss = 0.16322 :: 685.100 phrases/sec :: (3:3:17) hours left\n",
      "(4:57:0) step 6390/15121, epoch 0 Training Loss = 0.16303 :: 685.039 phrases/sec :: (3:2:51) hours left\n",
      "(4:57:28) step 6400/15121, epoch 0 Training Loss = 0.16339 :: 685.060 phrases/sec :: (3:2:22) hours left\n",
      "(4:57:55) step 6410/15121, epoch 0 Training Loss = 0.16388 :: 685.109 phrases/sec :: (3:1:53) hours left\n",
      "(4:58:24) step 6420/15121, epoch 0 Training Loss = 0.16332 :: 685.053 phrases/sec :: (3:1:26) hours left\n",
      "(4:58:50) step 6430/15121, epoch 0 Training Loss = 0.16323 :: 685.132 phrases/sec :: (3:0:57) hours left\n",
      "(4:59:17) step 6440/15121, epoch 0 Training Loss = 0.16552 :: 685.191 phrases/sec :: (3:0:27) hours left\n",
      "(4:59:44) step 6450/15121, epoch 0 Training Loss = 0.16418 :: 685.254 phrases/sec :: (2:59:58) hours left\n",
      "(5:0:13) step 6460/15121, epoch 0 Training Loss = 0.16325 :: 685.207 phrases/sec :: (2:59:31) hours left\n",
      "(5:0:41) step 6470/15121, epoch 0 Training Loss = 0.16314 :: 685.174 phrases/sec :: (2:59:4) hours left\n",
      "Saving model...\n",
      "(5:1:11) step 6480/15121, epoch 0 Training Loss = 0.16291 :: 685.227 phrases/sec :: (2:58:32) hours left\n",
      "(5:1:38) step 6490/15121, epoch 0 Training Loss = 0.16309 :: 685.239 phrases/sec :: (2:58:4) hours left\n",
      "(5:2:6) step 6500/15121, epoch 0 Training Loss = 0.16327 :: 685.237 phrases/sec :: (2:57:36) hours left\n",
      "(5:2:35) step 6510/15121, epoch 0 Training Loss = 0.16309 :: 685.207 phrases/sec :: (2:57:9) hours left\n",
      "(5:3:1) step 6520/15121, epoch 0 Training Loss = 0.16361 :: 685.260 phrases/sec :: (2:56:40) hours left\n",
      "(5:3:29) step 6530/15121, epoch 0 Training Loss = 0.16340 :: 685.270 phrases/sec :: (2:56:12) hours left\n",
      "(5:3:55) step 6540/15121, epoch 0 Training Loss = 0.16440 :: 685.355 phrases/sec :: (2:55:42) hours left\n",
      "(5:4:24) step 6550/15121, epoch 0 Training Loss = 0.16492 :: 685.341 phrases/sec :: (2:55:15) hours left\n",
      "(5:4:51) step 6560/15121, epoch 0 Training Loss = 0.16439 :: 685.374 phrases/sec :: (2:54:46) hours left\n",
      "(5:5:19) step 6570/15121, epoch 0 Training Loss = 0.16417 :: 685.373 phrases/sec :: (2:54:18) hours left\n",
      "(5:5:46) step 6580/15121, epoch 0 Training Loss = 0.16345 :: 685.421 phrases/sec :: (2:53:49) hours left\n",
      "(5:6:15) step 6590/15121, epoch 0 Training Loss = 0.16416 :: 685.361 phrases/sec :: (2:53:23) hours left\n",
      "(5:6:41) step 6600/15121, epoch 0 Training Loss = 0.16491 :: 685.458 phrases/sec :: (2:52:53) hours left\n",
      "(5:7:7) step 6610/15121, epoch 0 Training Loss = 0.16337 :: 685.523 phrases/sec :: (2:52:23) hours left\n",
      "(5:7:34) step 6620/15121, epoch 0 Training Loss = 0.16639 :: 685.570 phrases/sec :: (2:51:54) hours left\n",
      "(5:8:2) step 6630/15121, epoch 0 Training Loss = 0.16328 :: 685.592 phrases/sec :: (2:51:26) hours left\n",
      "(5:8:28) step 6640/15121, epoch 0 Training Loss = 0.16346 :: 685.672 phrases/sec :: (2:50:56) hours left\n",
      "(5:8:57) step 6650/15121, epoch 0 Training Loss = 0.16339 :: 685.642 phrases/sec :: (2:50:29) hours left\n",
      "(5:9:26) step 6660/15121, epoch 0 Training Loss = 0.16454 :: 685.577 phrases/sec :: (2:50:3) hours left\n",
      "(5:9:55) step 6670/15121, epoch 0 Training Loss = 0.16303 :: 685.517 phrases/sec :: (2:49:36) hours left\n",
      "(5:10:22) step 6680/15121, epoch 0 Training Loss = 0.16325 :: 685.540 phrases/sec :: (2:49:8) hours left\n",
      "(5:10:49) step 6690/15121, epoch 0 Training Loss = 0.16312 :: 685.596 phrases/sec :: (2:48:39) hours left\n",
      "(5:11:17) step 6700/15121, epoch 0 Training Loss = 0.16343 :: 685.615 phrases/sec :: (2:48:10) hours left\n",
      "(5:11:45) step 6710/15121, epoch 0 Training Loss = 0.16329 :: 685.589 phrases/sec :: (2:47:43) hours left\n",
      "(5:12:13) step 6720/15121, epoch 0 Training Loss = 0.16398 :: 685.552 phrases/sec :: (2:47:16) hours left\n",
      "(5:12:40) step 6730/15121, epoch 0 Training Loss = 0.16398 :: 685.611 phrases/sec :: (2:46:47) hours left\n",
      "(5:13:9) step 6740/15121, epoch 0 Training Loss = 0.16326 :: 685.581 phrases/sec :: (2:46:20) hours left\n",
      "(5:13:37) step 6750/15121, epoch 0 Training Loss = 0.16297 :: 685.523 phrases/sec :: (2:45:53) hours left\n",
      "(5:14:5) step 6760/15121, epoch 0 Training Loss = 0.16323 :: 685.554 phrases/sec :: (2:45:25) hours left\n",
      "(5:14:33) step 6770/15121, epoch 0 Training Loss = 0.16302 :: 685.528 phrases/sec :: (2:44:57) hours left\n",
      "(5:15:1) step 6780/15121, epoch 0 Training Loss = 0.16333 :: 685.508 phrases/sec :: (2:44:30) hours left\n",
      "(5:15:30) step 6790/15121, epoch 0 Training Loss = 0.16348 :: 685.449 phrases/sec :: (2:44:3) hours left\n",
      "(5:15:58) step 6800/15121, epoch 0 Training Loss = 0.16349 :: 685.453 phrases/sec :: (2:43:35) hours left\n",
      "(5:16:26) step 6810/15121, epoch 0 Training Loss = 0.16333 :: 685.449 phrases/sec :: (2:43:8) hours left\n",
      "(5:16:54) step 6820/15121, epoch 0 Training Loss = 0.16325 :: 685.451 phrases/sec :: (2:42:40) hours left\n",
      "(5:17:23) step 6830/15121, epoch 0 Training Loss = 0.16465 :: 685.402 phrases/sec :: (2:42:13) hours left\n",
      "(5:17:49) step 6840/15121, epoch 0 Training Loss = 0.16381 :: 685.473 phrases/sec :: (2:41:44) hours left\n",
      "(5:18:15) step 6850/15121, epoch 0 Training Loss = 0.16335 :: 685.556 phrases/sec :: (2:41:14) hours left\n",
      "(5:18:42) step 6860/15121, epoch 0 Training Loss = 0.16377 :: 685.607 phrases/sec :: (2:40:45) hours left\n",
      "(5:19:10) step 6870/15121, epoch 0 Training Loss = 0.16369 :: 685.612 phrases/sec :: (2:40:17) hours left\n",
      "(5:19:37) step 6880/15121, epoch 0 Training Loss = 0.16329 :: 685.667 phrases/sec :: (2:39:48) hours left\n",
      "(5:20:5) step 6890/15121, epoch 0 Training Loss = 0.16383 :: 685.671 phrases/sec :: (2:39:20) hours left\n",
      "(5:20:32) step 6900/15121, epoch 0 Training Loss = 0.16316 :: 685.716 phrases/sec :: (2:38:51) hours left\n",
      "(5:20:59) step 6910/15121, epoch 0 Training Loss = 0.16315 :: 685.742 phrases/sec :: (2:38:22) hours left\n",
      "(5:21:27) step 6920/15121, epoch 0 Training Loss = 0.16309 :: 685.743 phrases/sec :: (2:37:54) hours left\n",
      "(5:21:54) step 6930/15121, epoch 0 Training Loss = 0.16458 :: 685.760 phrases/sec :: (2:37:26) hours left\n",
      "(5:22:23) step 6940/15121, epoch 0 Training Loss = 0.16434 :: 685.699 phrases/sec :: (2:37:0) hours left\n",
      "(5:22:51) step 6950/15121, epoch 0 Training Loss = 0.16321 :: 685.717 phrases/sec :: (2:36:32) hours left\n",
      "(5:23:19) step 6960/15121, epoch 0 Training Loss = 0.16343 :: 685.736 phrases/sec :: (2:36:3) hours left\n",
      "(5:23:45) step 6970/15121, epoch 0 Training Loss = 0.16339 :: 685.813 phrases/sec :: (2:35:33) hours left\n",
      "(5:24:12) step 6980/15121, epoch 0 Training Loss = 0.16307 :: 685.833 phrases/sec :: (2:35:5) hours left\n",
      "(5:24:40) step 6990/15121, epoch 0 Training Loss = 0.16391 :: 685.871 phrases/sec :: (2:34:37) hours left\n",
      "(5:25:7) step 7000/15121, epoch 0 Training Loss = 0.16321 :: 685.864 phrases/sec :: (2:34:9) hours left\n",
      "(5:25:35) step 7010/15121, epoch 0 Training Loss = 0.16435 :: 685.892 phrases/sec :: (2:33:41) hours left\n",
      "(5:26:3) step 7020/15121, epoch 0 Training Loss = 0.16323 :: 685.871 phrases/sec :: (2:33:13) hours left\n",
      "(5:26:32) step 7030/15121, epoch 0 Training Loss = 0.16373 :: 685.802 phrases/sec :: (2:32:47) hours left\n",
      "(5:26:58) step 7040/15121, epoch 0 Training Loss = 0.16305 :: 685.900 phrases/sec :: (2:32:17) hours left\n",
      "(5:27:26) step 7050/15121, epoch 0 Training Loss = 0.16332 :: 685.918 phrases/sec :: (2:31:49) hours left\n",
      "(5:27:52) step 7060/15121, epoch 0 Training Loss = 0.16330 :: 685.985 phrases/sec :: (2:31:19) hours left\n",
      "(5:28:20) step 7070/15121, epoch 0 Training Loss = 0.16367 :: 685.981 phrases/sec :: (2:30:52) hours left\n",
      "(5:28:48) step 7080/15121, epoch 0 Training Loss = 0.16687 :: 685.990 phrases/sec :: (2:30:24) hours left\n",
      "(5:29:15) step 7090/15121, epoch 0 Training Loss = 0.16335 :: 686.024 phrases/sec :: (2:29:55) hours left\n",
      "(5:29:43) step 7100/15121, epoch 0 Training Loss = 0.16316 :: 686.028 phrases/sec :: (2:29:27) hours left\n",
      "(5:30:10) step 7110/15121, epoch 0 Training Loss = 0.16346 :: 686.034 phrases/sec :: (2:28:59) hours left\n",
      "(5:30:39) step 7120/15121, epoch 0 Training Loss = 0.16313 :: 685.996 phrases/sec :: (2:28:32) hours left\n",
      "Saving model...\n",
      "(5:31:8) step 7130/15121, epoch 0 Training Loss = 0.16462 :: 686.027 phrases/sec :: (2:28:1) hours left\n",
      "(5:31:38) step 7140/15121, epoch 0 Training Loss = 0.16317 :: 685.945 phrases/sec :: (2:27:35) hours left\n",
      "(5:32:7) step 7150/15121, epoch 0 Training Loss = 0.16307 :: 685.885 phrases/sec :: (2:27:9) hours left\n",
      "(5:32:35) step 7160/15121, epoch 0 Training Loss = 0.16316 :: 685.876 phrases/sec :: (2:26:41) hours left\n",
      "(5:33:1) step 7170/15121, epoch 0 Training Loss = 0.16320 :: 685.923 phrases/sec :: (2:26:12) hours left\n",
      "(5:33:28) step 7180/15121, epoch 0 Training Loss = 0.16368 :: 685.976 phrases/sec :: (2:25:43) hours left\n",
      "(5:33:57) step 7190/15121, epoch 0 Training Loss = 0.16379 :: 685.947 phrases/sec :: (2:25:16) hours left\n",
      "(5:34:24) step 7200/15121, epoch 0 Training Loss = 0.16332 :: 685.958 phrases/sec :: (2:24:48) hours left\n",
      "(5:34:51) step 7210/15121, epoch 0 Training Loss = 0.16327 :: 686.015 phrases/sec :: (2:24:19) hours left\n",
      "(5:35:18) step 7220/15121, epoch 0 Training Loss = 0.16329 :: 686.035 phrases/sec :: (2:23:51) hours left\n",
      "(5:35:47) step 7230/15121, epoch 0 Training Loss = 0.16354 :: 686.017 phrases/sec :: (2:23:23) hours left\n",
      "(5:36:15) step 7240/15121, epoch 0 Training Loss = 0.16335 :: 685.980 phrases/sec :: (2:22:56) hours left\n",
      "(5:36:43) step 7250/15121, epoch 0 Training Loss = 0.16312 :: 685.983 phrases/sec :: (2:22:28) hours left\n",
      "(5:37:11) step 7260/15121, epoch 0 Training Loss = 0.16286 :: 685.979 phrases/sec :: (2:22:1) hours left\n",
      "(5:37:39) step 7270/15121, epoch 0 Training Loss = 0.16339 :: 685.962 phrases/sec :: (2:21:33) hours left\n",
      "(5:38:7) step 7280/15121, epoch 0 Training Loss = 0.16331 :: 685.954 phrases/sec :: (2:21:5) hours left\n",
      "(5:38:35) step 7290/15121, epoch 0 Training Loss = 0.16339 :: 685.956 phrases/sec :: (2:20:37) hours left\n",
      "(5:39:3) step 7300/15121, epoch 0 Training Loss = 0.16442 :: 685.929 phrases/sec :: (2:20:10) hours left\n",
      "(5:39:32) step 7310/15121, epoch 0 Training Loss = 0.16544 :: 685.881 phrases/sec :: (2:19:43) hours left\n",
      "(5:40:0) step 7320/15121, epoch 0 Training Loss = 0.16339 :: 685.895 phrases/sec :: (2:19:15) hours left\n",
      "(5:40:26) step 7330/15121, epoch 0 Training Loss = 0.16375 :: 685.960 phrases/sec :: (2:18:46) hours left\n",
      "(5:40:54) step 7340/15121, epoch 0 Training Loss = 0.16318 :: 685.986 phrases/sec :: (2:18:17) hours left\n",
      "(5:41:23) step 7350/15121, epoch 0 Training Loss = 0.16475 :: 685.917 phrases/sec :: (2:17:51) hours left\n",
      "(5:41:50) step 7360/15121, epoch 0 Training Loss = 0.16344 :: 685.952 phrases/sec :: (2:17:22) hours left\n",
      "(5:42:19) step 7370/15121, epoch 0 Training Loss = 0.16324 :: 685.888 phrases/sec :: (2:16:56) hours left\n",
      "(5:42:46) step 7380/15121, epoch 0 Training Loss = 0.16341 :: 685.936 phrases/sec :: (2:16:27) hours left\n",
      "(5:43:13) step 7390/15121, epoch 0 Training Loss = 0.16334 :: 685.968 phrases/sec :: (2:15:59) hours left\n",
      "(5:43:41) step 7400/15121, epoch 0 Training Loss = 0.16312 :: 685.981 phrases/sec :: (2:15:30) hours left\n",
      "(5:44:9) step 7410/15121, epoch 0 Training Loss = 0.16498 :: 686.000 phrases/sec :: (2:15:2) hours left\n",
      "(5:44:37) step 7420/15121, epoch 0 Training Loss = 0.16356 :: 685.998 phrases/sec :: (2:14:34) hours left\n",
      "(5:45:4) step 7430/15121, epoch 0 Training Loss = 0.16310 :: 686.016 phrases/sec :: (2:14:6) hours left\n",
      "(5:45:32) step 7440/15121, epoch 0 Training Loss = 0.16322 :: 686.004 phrases/sec :: (2:13:38) hours left\n",
      "(5:46:1) step 7450/15121, epoch 0 Training Loss = 0.16421 :: 685.938 phrases/sec :: (2:13:12) hours left\n",
      "(5:46:30) step 7460/15121, epoch 0 Training Loss = 0.16340 :: 685.898 phrases/sec :: (2:12:45) hours left\n",
      "(5:46:58) step 7470/15121, epoch 0 Training Loss = 0.16314 :: 685.893 phrases/sec :: (2:12:17) hours left\n",
      "(5:47:25) step 7480/15121, epoch 0 Training Loss = 0.16341 :: 685.949 phrases/sec :: (2:11:48) hours left\n",
      "(5:47:51) step 7490/15121, epoch 0 Training Loss = 0.16394 :: 686.022 phrases/sec :: (2:11:19) hours left\n",
      "(5:48:18) step 7500/15121, epoch 0 Training Loss = 0.16318 :: 686.046 phrases/sec :: (2:10:50) hours left\n",
      "(5:48:46) step 7510/15121, epoch 0 Training Loss = 0.16306 :: 686.080 phrases/sec :: (2:10:22) hours left\n",
      "(5:49:13) step 7520/15121, epoch 0 Training Loss = 0.16344 :: 686.103 phrases/sec :: (2:9:53) hours left\n",
      "(5:49:41) step 7530/15121, epoch 0 Training Loss = 0.16334 :: 686.073 phrases/sec :: (2:9:26) hours left\n",
      "(5:50:10) step 7540/15121, epoch 0 Training Loss = 0.16351 :: 686.047 phrases/sec :: (2:8:59) hours left\n",
      "(5:50:39) step 7550/15121, epoch 0 Training Loss = 0.16288 :: 686.012 phrases/sec :: (2:8:31) hours left\n",
      "(5:51:8) step 7560/15121, epoch 0 Training Loss = 0.16371 :: 685.969 phrases/sec :: (2:8:4) hours left\n",
      "(5:51:36) step 7570/15121, epoch 0 Training Loss = 0.16346 :: 685.982 phrases/sec :: (2:7:36) hours left\n",
      "(5:52:4) step 7580/15121, epoch 0 Training Loss = 0.16363 :: 685.952 phrases/sec :: (2:7:9) hours left\n",
      "(5:52:32) step 7590/15121, epoch 0 Training Loss = 0.16305 :: 685.934 phrases/sec :: (2:6:41) hours left\n",
      "(5:53:1) step 7600/15121, epoch 0 Training Loss = 0.16339 :: 685.891 phrases/sec :: (2:6:14) hours left\n",
      "(5:53:28) step 7610/15121, epoch 0 Training Loss = 0.16340 :: 685.919 phrases/sec :: (2:5:46) hours left\n",
      "(5:53:55) step 7620/15121, epoch 0 Training Loss = 0.16310 :: 685.954 phrases/sec :: (2:5:17) hours left\n",
      "(5:54:23) step 7630/15121, epoch 0 Training Loss = 0.16311 :: 685.968 phrases/sec :: (2:4:49) hours left\n",
      "(5:54:51) step 7640/15121, epoch 0 Training Loss = 0.16344 :: 685.963 phrases/sec :: (2:4:22) hours left\n",
      "(5:55:20) step 7650/15121, epoch 0 Training Loss = 0.16411 :: 685.914 phrases/sec :: (2:3:55) hours left\n",
      "(5:55:47) step 7660/15121, epoch 0 Training Loss = 0.16479 :: 685.912 phrases/sec :: (2:3:27) hours left\n",
      "(5:56:17) step 7670/15121, epoch 0 Training Loss = 0.16302 :: 685.854 phrases/sec :: (2:3:0) hours left\n",
      "(5:56:45) step 7680/15121, epoch 0 Training Loss = 0.16296 :: 685.827 phrases/sec :: (2:2:33) hours left\n",
      "(5:57:13) step 7690/15121, epoch 0 Training Loss = 0.16333 :: 685.817 phrases/sec :: (2:2:5) hours left\n",
      "(5:57:43) step 7700/15121, epoch 0 Training Loss = 0.16303 :: 685.707 phrases/sec :: (2:1:39) hours left\n",
      "(5:58:11) step 7710/15121, epoch 0 Training Loss = 0.16315 :: 685.739 phrases/sec :: (2:1:11) hours left\n",
      "(5:58:40) step 7720/15121, epoch 0 Training Loss = 0.16326 :: 685.648 phrases/sec :: (2:0:45) hours left\n",
      "(5:59:8) step 7730/15121, epoch 0 Training Loss = 0.16394 :: 685.676 phrases/sec :: (2:0:17) hours left\n",
      "(5:59:35) step 7740/15121, epoch 0 Training Loss = 0.16338 :: 685.703 phrases/sec :: (1:59:48) hours left\n",
      "(6:0:2) step 7750/15121, epoch 0 Training Loss = 0.16358 :: 685.730 phrases/sec :: (1:59:20) hours left\n",
      "(6:0:31) step 7760/15121, epoch 0 Training Loss = 0.16340 :: 685.685 phrases/sec :: (1:58:53) hours left\n",
      "Saving model...\n",
      "(6:1:3) step 7770/15121, epoch 0 Training Loss = 0.16455 :: 685.674 phrases/sec :: (1:58:21) hours left\n",
      "(6:1:28) step 7780/15121, epoch 0 Training Loss = 0.16304 :: 685.697 phrases/sec :: (1:57:56) hours left\n",
      "(6:1:51) step 7790/15121, epoch 0 Training Loss = 0.16332 :: 685.800 phrases/sec :: (1:57:28) hours left\n",
      "(6:2:17) step 7800/15121, epoch 0 Training Loss = 0.16331 :: 685.792 phrases/sec :: (1:57:2) hours left\n",
      "(6:2:43) step 7810/15121, epoch 0 Training Loss = 0.16296 :: 685.796 phrases/sec :: (1:56:36) hours left\n",
      "(6:3:10) step 7820/15121, epoch 0 Training Loss = 0.16311 :: 685.782 phrases/sec :: (1:56:9) hours left\n",
      "(6:3:37) step 7830/15121, epoch 0 Training Loss = 0.16307 :: 685.792 phrases/sec :: (1:55:42) hours left\n",
      "(6:4:3) step 7840/15121, epoch 0 Training Loss = 0.16331 :: 685.810 phrases/sec :: (1:55:15) hours left\n",
      "(6:4:29) step 7850/15121, epoch 0 Training Loss = 0.16333 :: 685.862 phrases/sec :: (1:54:47) hours left\n",
      "(6:4:57) step 7860/15121, epoch 0 Training Loss = 0.16351 :: 685.833 phrases/sec :: (1:54:21) hours left\n",
      "(6:5:23) step 7870/15121, epoch 0 Training Loss = 0.16524 :: 685.869 phrases/sec :: (1:53:53) hours left\n",
      "(6:5:49) step 7880/15121, epoch 0 Training Loss = 0.16316 :: 685.917 phrases/sec :: (1:53:25) hours left\n",
      "(6:6:17) step 7890/15121, epoch 0 Training Loss = 0.16356 :: 685.892 phrases/sec :: (1:52:58) hours left\n",
      "(6:6:44) step 7900/15121, epoch 0 Training Loss = 0.16333 :: 685.898 phrases/sec :: (1:52:30) hours left\n",
      "(6:7:14) step 7910/15121, epoch 0 Training Loss = 0.16319 :: 685.817 phrases/sec :: (1:52:5) hours left\n",
      "(6:7:41) step 7920/15121, epoch 0 Training Loss = 0.16323 :: 685.805 phrases/sec :: (1:51:38) hours left\n",
      "(6:8:10) step 7930/15121, epoch 0 Training Loss = 0.16323 :: 685.747 phrases/sec :: (1:51:11) hours left\n",
      "(6:8:37) step 7940/15121, epoch 0 Training Loss = 0.16288 :: 685.751 phrases/sec :: (1:50:44) hours left\n",
      "(6:9:5) step 7950/15121, epoch 0 Training Loss = 0.16332 :: 685.737 phrases/sec :: (1:50:16) hours left\n",
      "(6:9:33) step 7960/15121, epoch 0 Training Loss = 0.16436 :: 685.743 phrases/sec :: (1:49:49) hours left\n",
      "(6:10:0) step 7970/15121, epoch 0 Training Loss = 0.16320 :: 685.744 phrases/sec :: (1:49:21) hours left\n",
      "(6:10:29) step 7980/15121, epoch 0 Training Loss = 0.16806 :: 685.695 phrases/sec :: (1:48:54) hours left\n",
      "(6:10:56) step 7990/15121, epoch 0 Training Loss = 0.16333 :: 685.734 phrases/sec :: (1:48:26) hours left\n",
      "(6:11:22) step 8000/15121, epoch 0 Training Loss = 0.16316 :: 685.774 phrases/sec :: (1:47:58) hours left\n",
      "(6:11:49) step 8010/15121, epoch 0 Training Loss = 0.16356 :: 685.793 phrases/sec :: (1:47:30) hours left\n",
      "(6:12:16) step 8020/15121, epoch 0 Training Loss = 0.16368 :: 685.858 phrases/sec :: (1:47:1) hours left\n",
      "(6:12:43) step 8030/15121, epoch 0 Training Loss = 0.16335 :: 685.850 phrases/sec :: (1:46:33) hours left\n",
      "(6:13:11) step 8040/15121, epoch 0 Training Loss = 0.16298 :: 685.879 phrases/sec :: (1:46:5) hours left\n",
      "(6:13:38) step 8050/15121, epoch 0 Training Loss = 0.16354 :: 685.897 phrases/sec :: (1:45:37) hours left\n",
      "(6:14:4) step 8060/15121, epoch 0 Training Loss = 0.16313 :: 685.964 phrases/sec :: (1:45:8) hours left\n",
      "(6:14:32) step 8070/15121, epoch 0 Training Loss = 0.16319 :: 685.961 phrases/sec :: (1:44:41) hours left\n",
      "(6:14:59) step 8080/15121, epoch 0 Training Loss = 0.16334 :: 685.972 phrases/sec :: (1:44:13) hours left\n",
      "(6:15:27) step 8090/15121, epoch 0 Training Loss = 0.16325 :: 685.987 phrases/sec :: (1:43:45) hours left\n",
      "(6:15:56) step 8100/15121, epoch 0 Training Loss = 0.16352 :: 685.925 phrases/sec :: (1:43:18) hours left\n",
      "(6:16:22) step 8110/15121, epoch 0 Training Loss = 0.16324 :: 685.968 phrases/sec :: (1:42:50) hours left\n",
      "(6:16:49) step 8120/15121, epoch 0 Training Loss = 0.16337 :: 686.009 phrases/sec :: (1:42:21) hours left\n",
      "(6:17:16) step 8130/15121, epoch 0 Training Loss = 0.16305 :: 686.038 phrases/sec :: (1:41:53) hours left\n",
      "(6:17:44) step 8140/15121, epoch 0 Training Loss = 0.16311 :: 686.037 phrases/sec :: (1:41:25) hours left\n",
      "(6:18:13) step 8150/15121, epoch 0 Training Loss = 0.16347 :: 685.975 phrases/sec :: (1:40:58) hours left\n",
      "(6:18:41) step 8160/15121, epoch 0 Training Loss = 0.16298 :: 685.958 phrases/sec :: (1:40:31) hours left\n",
      "(6:19:9) step 8170/15121, epoch 0 Training Loss = 0.16429 :: 685.934 phrases/sec :: (1:40:4) hours left\n",
      "(6:19:37) step 8180/15121, epoch 0 Training Loss = 0.16313 :: 685.939 phrases/sec :: (1:39:36) hours left\n",
      "(6:20:5) step 8190/15121, epoch 0 Training Loss = 0.16333 :: 685.948 phrases/sec :: (1:39:8) hours left\n",
      "(6:20:32) step 8200/15121, epoch 0 Training Loss = 0.16313 :: 685.964 phrases/sec :: (1:38:40) hours left\n",
      "(6:21:0) step 8210/15121, epoch 0 Training Loss = 0.16315 :: 685.947 phrases/sec :: (1:38:13) hours left\n",
      "(6:21:28) step 8220/15121, epoch 0 Training Loss = 0.16301 :: 685.912 phrases/sec :: (1:37:46) hours left\n",
      "(6:21:55) step 8230/15121, epoch 0 Training Loss = 0.16435 :: 685.953 phrases/sec :: (1:37:17) hours left\n",
      "(6:22:24) step 8240/15121, epoch 0 Training Loss = 0.16294 :: 685.893 phrases/sec :: (1:36:51) hours left\n",
      "(6:22:55) step 8250/15121, epoch 0 Training Loss = 0.16377 :: 685.770 phrases/sec :: (1:36:25) hours left\n",
      "(6:23:23) step 8260/15121, epoch 0 Training Loss = 0.16391 :: 685.738 phrases/sec :: (1:35:58) hours left\n",
      "(6:23:52) step 8270/15121, epoch 0 Training Loss = 0.16358 :: 685.685 phrases/sec :: (1:35:32) hours left\n",
      "(6:24:21) step 8280/15121, epoch 0 Training Loss = 0.16302 :: 685.633 phrases/sec :: (1:35:5) hours left\n",
      "(6:24:49) step 8290/15121, epoch 0 Training Loss = 0.16315 :: 685.641 phrases/sec :: (1:34:37) hours left\n",
      "(6:25:16) step 8300/15121, epoch 0 Training Loss = 0.16324 :: 685.685 phrases/sec :: (1:34:8) hours left\n",
      "(6:25:42) step 8310/15121, epoch 0 Training Loss = 0.16315 :: 685.754 phrases/sec :: (1:33:39) hours left\n",
      "(6:26:10) step 8320/15121, epoch 0 Training Loss = 0.16321 :: 685.726 phrases/sec :: (1:33:12) hours left\n",
      "(6:26:38) step 8330/15121, epoch 0 Training Loss = 0.16392 :: 685.737 phrases/sec :: (1:32:44) hours left\n",
      "(6:27:7) step 8340/15121, epoch 0 Training Loss = 0.16314 :: 685.677 phrases/sec :: (1:32:17) hours left\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 1\n",
    "batch_size =50\n",
    "neg_per = 25\n",
    "num_nearby = 20\n",
    "nearby_mod = 500\n",
    "num_steps = DH.num_steps(batch_size)\n",
    "total_step = 1\n",
    "save_interval = 30 * 60 # half hour in seconds\n",
    "save_time = time()\n",
    "#timing stuff\n",
    "start = time()\n",
    "fit_time = 0\n",
    "nearby_time = 0\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    DH.shuffle_data()\n",
    "    for step , batch in enumerate(DH.batches(batch_size, offset=6780, neg_per=neg_per)):\n",
    "#         print(batch)\n",
    "        t0 = time()\n",
    "        loss = drnn.partial_fit(*batch)\n",
    "        fit_time = (fit_time * float(total_step) +  time() - t0) / (total_step + 1) # running average\n",
    "        if step % 10 == 0:\n",
    "            m,s = divmod(time()-start, 60)\n",
    "            h,m = divmod(m, 60)\n",
    "            left = time_left(num_epochs, num_steps, fit_time, nearby_time, start, nearby_mod)\n",
    "            ml,sl = divmod(left, 60)\n",
    "            hl,ml = divmod(ml, 60)\n",
    "            pps = batch_size*(neg_per + 1) / fit_time \n",
    "            print(\"(%i:%i:%i) step %i/%i, epoch %i Training Loss = %1.5f :: %0.3f phrases/sec :: (%i:%i:%i) hours left\" \n",
    "                  % (h,m,s, step, num_steps, epoch, loss, pps, hl, ml, sl))\n",
    "        if (total_step-1) % nearby_mod == 0: # do one right away so we get a good timing estimate\n",
    "            t0 = time()\n",
    "            run_validation_test(drnn, DH, num_nearby)\n",
    "            nearby_time = (nearby_time * float(total_step) + time() - t0) / (total_step + 1) # running average\n",
    "\n",
    "        if (time() - save_time) > save_interval:\n",
    "            print(\"Saving model...\")\n",
    "            drnn.checkpoint()\n",
    "            save_time = time()\n",
    "        total_step +=1"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Batch size: 100, 25 neg per :: ~ 602.887 phrases/sec\n",
    "Batch size: 50, 25 neg per :: ~ 660 phrases/sec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "Top 50 closest phrases to 'purchased land in district was on outskirts of'\n",
      "0: 0.998 : 'of most spent in research meetings in countries including France Switzerland Romania Turkey Britain Belgium Germany Denmark Sweden Greece'\n",
      "1: 0.998 : 'model model model model fidelity model fidelity model model model model model model mapped model extraction space space mapping'\n",
      "2: 0.998 : '<OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> McCormack <OOV> <OOV> Moore <OOV> <OOV> <OOV> <OOV> <OOV> <OOV>'\n",
      "3: 0.998 : '<OOV> <OOV> Todd <OOV> <OOV> <OOV> <OOV> <OOV> Music King <OOV> <OOV> <OOV> <OOV> Gibbons Black <OOV> Orchestra'\n",
      "4: 0.997 : 'of program with accordance in development of plans statements of <OOV> Party decisions of Council of Ministers of'\n",
      "5: 0.997 : '<OOV> <OOV> <OOV> <OOV> Smith <OOV> Clement <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> Powell <OOV>'\n",
      "6: 0.997 : 'in recorded detection in <OOV> of part as used asked series of questions pertaining to issue under'\n",
      "7: 0.997 : 'found in Spain Italy France <OOV> Germany Austria Switzerland Poland Republic Slovakia Hungary Romania Slovenia Croatia'\n",
      "8: 0.997 : 'around parties in workers of organization Department under emerged given <OOV> relative to activities of parties affiliated with'\n",
      "9: 0.997 : 'India Malaysia Iran Armenia Georgia Turkey Kazakhstan Russia Spain Kingdom Germany France Italy Austria Norway Poland'\n",
      "10: 0.996 : 'of audience have travelled submitted report showed based upon documents about espionage in favour of Kingdom of'\n",
      "11: 0.996 : 'as <OOV> corporation institution dedicated educate with emphasis on achieving maintaining <OOV> aiming develop College holistic <OOV> in'\n",
      "12: 0.996 : 'of problems known due development of lenses work with plane of <OOV> device for projecting images used as'\n",
      "13: 0.996 : 'launched in cooperation with <OOV> allows goods sold by parties collected in location allows collect at'\n",
      "14: 0.996 : 'innovations water Environment promotion education to Right to Right by are bring empowerment medicine care issues'\n",
      "15: 0.996 : 'education to access agency abilities independence as issues to tied created <OOV> as poverty <OOV> issues of'\n",
      "16: 0.996 : 'seen Mills accompanied by musicians as Browne Gibbons <OOV> <OOV> Lewis <OOV> <OOV> <OOV> <OOV>'\n",
      "17: 0.996 : 'In provided opinion supporting argument ceased be eligible remain day accepted appointment as Ambassador to Ireland is office of'\n",
      "18: 0.996 : '<OOV> Head Agency Social of Head <OOV> <OOV> in staff was <OOV> Head Head of'\n",
      "19: 0.996 : 'organized by M channel owned by E M Channel with support from Entertainment one of labels in Korea houses groups including Ace'\n",
      "20: 0.996 : 'of knight <OOV> of Order of Cross of Chamberlain Knight was knight of Order of'\n",
      "21: 0.995 : 'processes for store was subject of persecution from IFPI entity subject of <OOV> had with'\n",
      "22: 0.995 : 'within <OOV> for important are is in order maintain position as center of excellence in'\n",
      "23: 0.995 : 'bass <OOV> with <OOV> including groups of series with made included <OOV> <OOV> Turner <OOV>'\n",
      "24: 0.995 : 'for process of founders of one after named given to those made contribution toward to'\n",
      "25: 0.995 : 'training <OOV> training education education <OOV> are by Right to Right to education promotion Environment'\n",
      "26: 0.995 : 'for Committee Law Law of <OOV> for committee chaired serving for Literature <OOV> for'\n",
      "27: 0.995 : 'including <OOV> in nature of Corpus <OOV> Prohibition <OOV> with in territory for for'\n",
      "28: 0.995 : 'Russia Spain Kingdom Germany France Italy Austria Norway Poland Ukraine Belgium Bosnia Herzegovina Hungary'\n",
      "29: 0.995 : 'has offices in Toronto Vancouver City California Caracas Jakarta Kong <OOV> Dhaka Singapore Casablanca'\n",
      "30: 0.995 : '<OOV> <OOV> <OOV> Montana Sean <OOV> <OOV> Wayne Brown Khalifa <OOV> <OOV> Drake <OOV>'\n",
      "31: 0.995 : '<OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> McCormack <OOV> <OOV> Moore <OOV>'\n",
      "32: 0.995 : 'provides to students in communities of Wilson <OOV> <OOV> <OOV> <OOV> <OOV> Bassett <OOV>'\n",
      "33: 0.995 : 'for team of member is coached trained by brother medalist in division at Olympics in'\n",
      "34: 0.995 : 'cover task for lens creating problems known due development of lenses work with plane of'\n",
      "35: 0.995 : 'of study conducting with charged <OOV> <OOV> reduce <OOV> among youth <OOV> to authority of courts'\n",
      "36: 0.995 : 'of Faculty Faculty is faculty comprises schools departments provide array of programmes in medicine nursing'\n",
      "37: 0.995 : 'in Bachelor colleges of functioning quality on report submitted approved during meetings held on'\n",
      "38: 0.995 : 'For <OOV> around ball got <OOV> in tree coin stolen from <OOV> <OOV> of'\n",
      "39: 0.995 : 'Lyon December on died was priest member of <OOV> teacher director of school in'\n",
      "40: 0.995 : 'on born <OOV> died was priest member of <OOV> teacher director of school in'\n",
      "41: 0.995 : 'in administration of accusations based upon documents about espionage in favour of Kingdom of'\n",
      "42: 0.995 : 'in found Leone Liberia Coast Ghana Togo <OOV> Nigeria Cameroon Gabon Republic of Congo'\n",
      "43: 0.994 : 'services improving <OOV> <OOV> <OOV> reduce <OOV> among youth <OOV> to authority of courts <OOV> to'\n",
      "44: 0.994 : 'in schools at teaching After took for <OOV> excluded did For Children at'\n",
      "45: 0.994 : 'of recordings made for Library Yale Harvard Stanford University University University University of'\n",
      "46: 0.994 : 'for <OOV> reason is father to connected loves ; wife sister sees as <OOV> to climbing'\n",
      "47: 0.994 : '<OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> <OOV> McCormack <OOV> <OOV> Moore <OOV>'\n",
      "48: 0.994 : 'have range of accessories as <OOV> Bear <OOV> <OOV> <OOV> <OOV> <OOV> Hat'\n",
      "49: 0.994 : 'of president Bose grandfather included family to Born studied earned degree from University studied'\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "run_validation_test(drnn, DH, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}