Dependency RNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dependency RNNs\n",
    "\n",
    "To try to embed words and dependency relations s.t. they are predictive of relations, consider using a simple RNN or LSTM/GRU to predict the target noun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import collections\n",
    "import random\n",
    "from time import time\n",
    "\n",
    "# plot settings\n",
    "% matplotlib inline\n",
    "# print(plt.rcParams.keys())\n",
    "plt.rcParams['figure.figsize'] = (16,9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import data_helper as dh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# reload(dh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# reload(tf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset Curation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "semeval_train, semeval_valid = dh.load_semeval_data()\n",
    "label2int = dh.label2int\n",
    "int2label = {v:k for (k,v) in label2int.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size: 19300\n",
      "Dependency vocab size: 45\n"
     ]
    }
   ],
   "source": [
    "# create a vocab and dependency vocab\n",
    "vocab_size = 100000\n",
    "(vocab, vocab2int, int2vocab, vocab_dist) = (dh.create_vocab_from_data(\n",
    "                                                  [datum[0] for datum in (semeval_train['x']+semeval_valid['x'])],\n",
    "                                                  vocab_limit=vocab_size,\n",
    "                                                  filter_oov=False))\n",
    "# assert len(vocab) == vocab_size, \"We don't have enough embeddings for those\"\n",
    "vocab_set = set(vocab)\n",
    "print(\"Vocab size: %i\" % len(vocab_set))\n",
    "(dep_vocab, dep2int, int2dep, dep_dist) = (dh.create_vocab_from_data(\n",
    "                                                  [datum[0] for datum in (semeval_train['x']+semeval_valid['x'])],\n",
    "                                                  vocab_limit=50,\n",
    "                                                  filter_oov=False,\n",
    "                                                  dep=True))\n",
    "dep_set = set(dep_vocab)\n",
    "print(\"Dependency vocab size: %i\" %len(dep_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the system as described above has its greatest application in an arrayed configuration of antenna elements\n"
     ]
    }
   ],
   "source": [
    "print(semeval_train['x'][0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before OOV: 7109 training sentences and 891 validation\n",
      "After OOV: 7109 training sentences and 891 validation\n"
     ]
    }
   ],
   "source": [
    "print(\"Before OOV: %i training sentences and %i validation\" % (len(semeval_train['x']), len(semeval_valid['x'])))\n",
    "semeval_train['x'], semeval_train['y'] = zip(*[ (datum, label) for (datum, label) \n",
    "                                                in zip(semeval_train['x'], semeval_train['y'])\n",
    "                                                if not dh.is_oov(datum[0], vocab_set)])\n",
    "semeval_valid['x'], semeval_valid['y'] = zip(*[ (datum, label) for (datum, label) \n",
    "                                                in zip(semeval_valid['x'], semeval_valid['y'])\n",
    "                                                if not dh.is_oov(datum[0], vocab_set)])\n",
    "print(\"After OOV: %i training sentences and %i validation\" % (len(semeval_train['x']), len(semeval_valid['x'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR: This sentence has no common dependency ancestor.  It was probably parsed incorrectly. SKIPPING\n",
      "the doctor implanted an injection into three vertical glabellar furrows\n",
      "[the , doctor , implanted , an , injection , into , three , vertical , glabellar , furrows]\n",
      "4 9 [injection , implanted ] [furrows]\n",
      "ERROR: This sentence has no common dependency ancestor.  It was probably parsed incorrectly. SKIPPING\n",
      "this speed bag plaform is made of heavy duty powder coated steel that secures easily to a wood stud, brick or concrete\n",
      "[this , speed , bag , plaform , is , made , of , heavy , duty , powder , coated , steel , that , secures , easily , to , a , wood , stud, , , brick , or , concrete]\n",
      "3 11 [plaform , made ] [steel ]\n",
      "ERROR: This sentence has no common dependency ancestor.  It was probably parsed incorrectly. SKIPPING\n",
      "also, very rarely, hypnotherapy leads to the development of 'false memories' fabricated by the unconscious mind these are called confabulations\n",
      "[also, , , very , rarely, , , hypnotherapy , leads , to , the , development , of , ', false , memories, ' , fabricated , by , the , unconscious , mind , these , are , called , confabulations]\n",
      "10 15 [of , development , to , leads ] [fabricated , called ]\n",
      "\n",
      "After SDP conversion: 14214 training sentences and 1780 validation\n"
     ]
    }
   ],
   "source": [
    "train_x, train_y = dh.convert_semeval_to_sdps(semeval_train['x'], semeval_train['y'], \n",
    "                                           vocab2int, dep2int,\n",
    "                                           int2label, label2int,\n",
    "                                           include_reverse=True,\n",
    "                                           print_check=False)\n",
    "valid_x, valid_y = dh.convert_semeval_to_sdps(semeval_valid['x'], semeval_valid['y'], \n",
    "                                           vocab2int, dep2int,\n",
    "                                           int2label, label2int,\n",
    "                                           include_reverse=True, \n",
    "                                           print_check=False)\n",
    "print(\"\\nAfter SDP conversion: %i training sentences and %i validation\" \n",
    "      % (len(train_x), len(valid_x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3352, 5, 1445, 23, 1445, 5, 3352, 1, 2151] 0\n"
     ]
    }
   ],
   "source": [
    "# quick taste of data:\n",
    "print(train_x[0], train_y[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model instantiation and training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Sequence Length: 28\n"
     ]
    }
   ],
   "source": [
    "max_sequence_len = max([len(x) for x in (train_x+valid_x)])\n",
    "print(\"Max Sequence Length: %i\" % max_sequence_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class DPRNN(object):\n",
    "    \"\"\" Encapsulation of the dependency RNN lang model\n",
    "    \n",
    "    Largely inspired by https://github.com/tensorflow/tensorflow/blob/master/tensorflow/models/rnn/ptb/ptb_word_lm.py\n",
    "    \"\"\"\n",
    "    def __init__(self, config):\n",
    "        self.batch_size = config['batch_size']\n",
    "        self.max_num_steps = config['max_num_steps']\n",
    "        self.embedding_size = config['embedding_size']\n",
    "        self.hidden_size = config['hidden_size']\n",
    "        self.max_grad_norm = config['max_grad_norm']\n",
    "        self.vocab_dist = config['vocab_dist']\n",
    "        self.vocab_size = len(self.vocab_dist)\n",
    "        self.nce_num_samples = config['nce_num_samples']\n",
    "        self.checkpoint_prefix = config['checkpoint_prefix']\n",
    "        \n",
    "        self.initializer = tf.random_uniform_initializer(-1., 1.)\n",
    "        \n",
    "        with tf.name_scope(\"Forward\"):\n",
    "            self._build_forward_graph()\n",
    "        with tf.name_scope(\"Backward\"):\n",
    "            self._build_train_graph()\n",
    "        with tf.name_scope(\"Predict\"):\n",
    "            self._build_predict_graph()\n",
    "        \n",
    "        self.saver = tf.train.Saver(tf.all_variables())\n",
    "            \n",
    "        self.session = tf.InteractiveSession()\n",
    "        self.session.run(tf.initialize_all_variables())        \n",
    "        self.summary_writer = tf.train.SummaryWriter(\"tensor_summaries/\", self.session.graph_def)\n",
    "\n",
    "        \n",
    "    def _build_forward_graph(self):\n",
    "        # TODO: Add summaries\n",
    "        # input tensor of zero padded indices to get to max_num_steps\n",
    "        # None allows for variable batch sizes\n",
    "        with tf.name_scope(\"Inputs\"):\n",
    "            self._input_data = tf.placeholder(tf.int32, [None, self.max_num_steps])\n",
    "            self._input_labels = tf.placeholder(tf.int32, [None, self.max_num_steps])\n",
    "            self._input_lengths = tf.placeholder(tf.int32, [None, 1]) \n",
    "            batch_size = tf.shape(self._input_lengths)[0]\n",
    "        \n",
    "        with tf.name_scope(\"Word_Embeddings\"):\n",
    "            self._word_embeddings = tf.get_variable(\"word_embeddings\", \n",
    "                                                    [self.vocab_size, self.embedding_size],\n",
    "                                                    dtype=tf.float32)\n",
    "        \n",
    "            input_embeds = tf.nn.embedding_lookup(self._word_embeddings, self._input_data)\n",
    "            print(input_embeds.get_shape())\n",
    "            # TODO: Add dropout to embeddings\n",
    "        \n",
    "        with tf.name_scope(\"RNN\"):\n",
    "            # start off with the most basic configuration\n",
    "            self.cell = tf.nn.rnn_cell.BasicRNNCell(self.hidden_size)#, input_size=self.embedding_size)\n",
    "            # TODO: Add Dropout wrapper\n",
    "            # TODO: Make it multilevel\n",
    "            self._initial_state = self.cell.zero_state(batch_size, tf.float32)\n",
    "            inputs = [ tf.squeeze(input_, [1]) for input_ in tf.split(1, self.max_num_steps, input_embeds)]\n",
    "\n",
    "            outputs, state = tf.nn.rnn(self.cell, inputs, \n",
    "                                           sequence_length=tf.squeeze(self._input_lengths, [1]),\n",
    "                                           initial_state=self._initial_state)\n",
    "            self._final_state = state\n",
    "            output = tf.reshape(tf.concat(1, outputs), [-1, self.hidden_size])\n",
    "        \n",
    "        # now get unnormalized predictions\n",
    "        with tf.name_scope(\"Softmax\"):\n",
    "            self._softmax_w = tf.get_variable(\"softmax_w\", [self.vocab_size, self.hidden_size, ], dtype=tf.float32)\n",
    "            self._softmax_b = tf.get_variable(\"softmax_b\", [self.vocab_size], dtype=tf.float32)\n",
    "#             self._logits = tf.matmul(output, self._softmax_w) + self._softmax_b\n",
    "        \n",
    "        # loss is classification cross entropy for each word\n",
    "        # NOTE: that where we have 0 outputs the loss contribution \n",
    "        #       is only the log(bias) of that vocab word\n",
    "        with tf.name_scope(\"Cost\"):\n",
    "            # Softmax loss is SUPER slow. Softmaxes of 20,000 words is very slow\n",
    "#             flat_labels = tf.reshape(self._input_labels, [-1])\n",
    "#             self._loss = tf.nn.seq2seq.sequence_loss_by_example(\n",
    "#                             [self._logits],\n",
    "#                             [flat_labels], # [num_steps*batch_size]\n",
    "#                             [tf.ones_like(flat_labels, dtype=tf.float32)]) # weights to for each example\n",
    "            # Use NCE instead\n",
    "            flat_labels = tf.reshape(self._input_labels, [-1, 1])\n",
    "            sampler = tf.nn.fixed_unigram_candidate_sampler(true_classes=tf.to_int64(flat_labels), \n",
    "                                                            num_true=1, \n",
    "                                                            num_sampled=self.nce_num_samples, \n",
    "                                                            unique=True, \n",
    "                                                            range_max=self.vocab_size,\n",
    "                                                            distortion=.75,\n",
    "                                                            num_reserved_ids=0,\n",
    "                                                            unigrams=self.vocab_dist,\n",
    "                                                            seed=0,\n",
    "                                                            name=None)\n",
    "            self._cost = tf.reduce_mean(tf.nn.nce_loss(weights=self._softmax_w, \n",
    "                                             biases=self._softmax_b, \n",
    "                                             inputs=output, \n",
    "                                             labels=tf.to_int64(flat_labels), \n",
    "                                             num_sampled=self.nce_num_samples, \n",
    "                                             num_classes=self.vocab_size, \n",
    "                                             num_true=1, \n",
    "                                             sampled_values=sampler, \n",
    "                                             remove_accidental_hits=False, \n",
    "                                             name='nce_loss'))\n",
    "            \n",
    "            self._train_cost_summary = tf.merge_summary([tf.scalar_summary(\"Train_NCE_Loss\", self._cost)])\n",
    "            self._valid_cost_summary = tf.merge_summary([tf.scalar_summary(\"Validation_NCE_Loss\", self._cost)])\n",
    "            # add another dimension so tf doesn't get made about NoneType tensors\n",
    "#             self._cost = tf.reduce_mean(loss)\n",
    "#             print(loss.get_shape(), self._cost.get_shape())\n",
    "        \n",
    "    def _build_train_graph(self):\n",
    "        with tf.name_scope(\"Trainer\"):\n",
    "            self._global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
    "            self._lr = tf.Variable(1.0, trainable=False)\n",
    "            self._optimizer = tf.train.AdagradOptimizer(self._lr)\n",
    "            \n",
    "            # clip and apply gradients\n",
    "            grads_and_vars = self.optimizer.compute_gradients(self._cost)\n",
    "#             for gv in grads_and_vars:\n",
    "#                 print(gv, gv[1] is self._cost)\n",
    "            clipped_grads_and_vars = [(tf.clip_by_norm(gv[0], self.max_grad_norm), gv[1]) \n",
    "                                      for gv in grads_and_vars if gv[0] is not None] # clip_by_norm doesn't like None\n",
    "            \n",
    "            with tf.name_scope(\"Summaries\"):\n",
    "                grad_summaries = []\n",
    "                for g, v in grads_and_vars:\n",
    "                    if g is not None:\n",
    "                        grad_hist_summary = tf.histogram_summary(\"{}/grad/hist\".format(v.name), g)\n",
    "                        sparsity_summary = tf.scalar_summary(\"{}/grad/sparsity\".format(v.name), tf.nn.zero_fraction(g))\n",
    "                        grad_summaries.append(grad_hist_summary)\n",
    "                        grad_summaries.append(sparsity_summary)\n",
    "                self._grad_summaries = tf.merge_summary(grad_summaries)\n",
    "            self._train_op = self._optimizer.apply_gradients(clipped_grads_and_vars, global_step=self._global_step)\n",
    "            \n",
    "    def _build_predict_graph(self):\n",
    "        tf.get_variable_scope().reuse_variables()\n",
    "        with tf.name_scope(\"Inputs\"):\n",
    "            self._predict_inputs = tf.placeholder(tf.int32, [None, self.max_num_steps])\n",
    "            self._predict_lengths = tf.placeholder(tf.int32, [None, 1])\n",
    "        \n",
    "        with tf.name_scope(\"Word_Embeddings\"):\n",
    "            predict_embeds = tf.nn.embedding_lookup(self._word_embeddings, self._predict_inputs)\n",
    "        \n",
    "        with tf.name_scope(\"RNN\"):\n",
    "            predict_inputs = [tf.squeeze(input_, [1]) for input_\n",
    "                              in tf.split(1, self.max_num_steps, predict_embeds)]\n",
    "            _initial_state = self.cell.zero_state(tf.shape(self._predict_lengths)[0], tf.float32)\n",
    "            predict_outputs, predict_state = tf.nn.rnn(self.cell, predict_inputs, \n",
    "                                                       sequence_length=tf.squeeze(self._predict_lengths, [1]), \n",
    "                                                       initial_state=_initial_state)\n",
    "        with tf.name_scope(\"Softmax_and_predict\"):\n",
    "            predict_outputs = tf.reshape(tf.concat(1,predict_outputs), [-1, self.hidden_size])\n",
    "            predict_logits = tf.matmul(predict_outputs, self.softmax_w, transpose_b=True) + self._softmax_b\n",
    "            self._predicted_dists = tf.nn.softmax(predict_logits)\n",
    "            self._predictions = tf.reshape(tf.argmax(predict_logits, 1), [-1, self.max_num_steps])\n",
    "            \n",
    "    def partial_fit(self, batch_x, batch_y, batch_seq_lens):\n",
    "        \"\"\"Fit a mini-batch\n",
    "        \n",
    "        Expects a batch_x: [self.batch_size, self.max_num_steps]\n",
    "                  batch_y: the same\n",
    "                  batch_seq_lens: [self.batch_size]\n",
    "                  \n",
    "        Returns average batch perplexity\n",
    "        \"\"\"\n",
    "        feed = {self._input_data:batch_x, \n",
    "                self._input_labels:batch_y,\n",
    "                self._input_lengths:batch_seq_lens}\n",
    "#         print(feed)\n",
    "#         print(batch_x, batch_y, batch_seq_lens)\n",
    "        cost, _, g_summaries, c_summary = self.session.run([self._cost, self._train_op, \n",
    "                                             self._grad_summaries,\n",
    "                                             self._train_cost_summary], feed_dict=feed)\n",
    "        self.summary_writer.add_summary(g_summaries)\n",
    "        self.summary_writer.add_summary(c_summary)\n",
    "#         print(cost)\n",
    "        perp = cost# np.exp(cost)\n",
    "        return perp\n",
    "    \n",
    "    def validation_cost(self, batch_x, batch_y, batch_seq_lens):\n",
    "        \"\"\"Run a forward pass of the RNN on a batch of validation data \n",
    "        and report the perplexity\n",
    "        \"\"\"\n",
    "        feed = {self._input_data:batch_x, \n",
    "                self._input_labels:batch_y,\n",
    "                self._input_lengths:batch_seq_lens}\n",
    "#         print(feed)\n",
    "        cost, valid_cost_summary = self.session.run([self._cost, self._valid_cost_summary], feed_dict=feed)\n",
    "        self.summary_writer.add_summary(valid_cost_summary)\n",
    "        perp = cost #np.exp(cost)\n",
    "        return perp\n",
    "        \n",
    "    def predict(self, sequences, seq_lens, return_probs=False):\n",
    "        if return_probs:\n",
    "            predictions, distributions = self.session.run([self._predictions, self._predicted_dists],\n",
    "                                                          {self._predict_inputs:sequences,\n",
    "                                                           self._predict_lengths:seq_lens})\n",
    "            distributions = distributions.reshape([sequences.shape[0], sequences.shape[1], -1])\n",
    "            pred_list = []\n",
    "            dist_list = []\n",
    "            for i, seq_len in enumerate(seq_lens):\n",
    "                pred_list.append(list(predictions[i, :seq_len]))\n",
    "                dist_list.append([distributions[i,j,:] for j in range(seq_len)])\n",
    "            return pred_list, dist_list\n",
    "        \n",
    "        else:\n",
    "            predictions = self.session.run(self._predictions,\n",
    "                                           {self._predict_inputs:sequences,\n",
    "                                            self._predict_lengths:seq_lens})\n",
    "            pred_list = []\n",
    "            for i, seq_len in enumerate(seq_lens):\n",
    "                pred_list.append(list(predictions[i, :seq_len])) \n",
    "            return pred_list\n",
    "            \n",
    "    def checkpoint(self):\n",
    "        self.saver.save(self.session, self.checkpoint_prefix, global_step=self._global_step)\n",
    "        \n",
    "    def __repr__(self):\n",
    "        return \"<DPNN: Embed:%i, Hidden:%i, V:%i>\" % (self.embedding_size, self.hidden_size, self.vocab_size)\n",
    "        \n",
    "    @property\n",
    "    def input_data(self):\n",
    "        return self._input_data\n",
    "    \n",
    "    @property\n",
    "    def labels(self):\n",
    "        return self._labels\n",
    "    \n",
    "    @property\n",
    "    def input_lengths(self):\n",
    "        return self._input_lengths\n",
    "    \n",
    "    @property\n",
    "    def initial_state(self):\n",
    "        return self._initial_state\n",
    "    \n",
    "    @property\n",
    "    def final_state(self):\n",
    "        return self._final_state\n",
    "    \n",
    "    @property\n",
    "    def word_embeddings(self):\n",
    "        return self._word_embeddings\n",
    "    \n",
    "    @property\n",
    "    def softmax_w(self):\n",
    "        return self._softmax_w\n",
    "    \n",
    "    @property\n",
    "    def softmax_b(self):\n",
    "        return self._softmax_b\n",
    "    \n",
    "    @property\n",
    "    def loss(self):\n",
    "        return self._loss\n",
    "    \n",
    "    @property\n",
    "    def cost(self):\n",
    "        return self._cost\n",
    "    \n",
    "    @property\n",
    "    def lr(self):\n",
    "        return self._lr\n",
    "    \n",
    "    @property\n",
    "    def optimizer(self):\n",
    "        return self._optimizer\n",
    "    @property\n",
    "    def train_op(self):\n",
    "        return self._train_op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Sequence Length: 28\n"
     ]
    }
   ],
   "source": [
    "max_sequence_len = max([len(x) for x in (train_x+valid_x)])\n",
    "print(\"Max Sequence Length: %i\" % max_sequence_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# fake easy dataset\n",
    "# easy_sentences = [dh.nlp(u\"the quick brown fox jumps over the lazy dog\")]*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# (vocab, vocab2int, int2vocab, vocab_dist) = dh.create_vocab_from_data(easy_sentences)\n",
    "# # print(vocab_dist)\n",
    "# easy_data = dh.sentences_to_indices(easy_sentences, vocab2int)\n",
    "# easy_data[::2].pop()\n",
    "# random.seed(0)\n",
    "# for i, s in enumerate(easy_data):\n",
    "#     r = random.randint(0,3)\n",
    "#     if r:\n",
    "#         easy_data[i] = s[:-r]\n",
    "# max_seq_len = max([len(d) for d in easy_data])\n",
    "# print(\"Max seq len: %i\" % max_seq_len)\n",
    "# # for i, s in enumerate(easy_data):\n",
    "# #     print(i, s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?, 28, 200)\n",
      "No need for new setup\n",
      "<DPNN: Embed:200, Hidden:200, V:19300>\n"
     ]
    }
   ],
   "source": [
    "# tf.get_default_session().close()\n",
    "tf.reset_default_graph()\n",
    "k = 20\n",
    "config = {\n",
    "    'batch_size':100,\n",
    "    'max_num_steps':max_sequence_len,\n",
    "    'embedding_size':200,\n",
    "    'hidden_size':200,\n",
    "    'max_grad_norm':3.,\n",
    "    'vocab_dist':vocab_dist,\n",
    "    'nce_num_samples':25,\n",
    "    'checkpoint_prefix':'checkpoints/dprnn_test1'\n",
    "}\n",
    "try:\n",
    "    drnn = DPRNN(config)\n",
    "    print(\"No need for new setup\")\n",
    "except:\n",
    "    try:\n",
    "        tf.reset_default_graph()\n",
    "    except:\n",
    "        pass\n",
    "    try:\n",
    "        tf.get_default_session().close()\n",
    "    except:\n",
    "        pass\n",
    "    print(\"Needed a new setup\")\n",
    "    drnn = DPRNN(config)\n",
    "print(drnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sequences_to_matrix(list_of_lists, max_seq_len=None):\n",
    "    lengths = np.array([len(list_) for list_ in list_of_lists]).reshape([-1, 1])\n",
    "    if max_seq_len:\n",
    "        assert max_seq_len >= max(lengths), \"Currently cant force sequnece lengths to be shorter than max list len\"\n",
    "    else:\n",
    "        max_seq_len = max(lengths)\n",
    "    matrix = np.zeros([len(list_of_lists), max_seq_len])\n",
    "    for i, list_ in enumerate(list_of_lists):\n",
    "        matrix[i, :len(list_)] = list_\n",
    "    return matrix, lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_lang_model_batch(offset, batch_size, data_x, max_seq_len=None):\n",
    "    \"\"\"Expects the data as list of lists of indices\n",
    "    \n",
    "    Converts them to matrices of indices, lang model labels, and lengths\"\"\"\n",
    "    start = offset*batch_size\n",
    "    end = start + batch_size\n",
    "    if end > len(data_x):\n",
    "        end = len(data_x)\n",
    "        print(\"Not full batch\")\n",
    "    batch = data_x[start:end]\n",
    "    inputs = [ seq[:-1] for seq in batch ]\n",
    "    labels = [ seq[1:] for seq in batch ]\n",
    "    input_mat, len_vec = sequences_to_matrix(inputs, max_seq_len=max_sequence_len)\n",
    "    label_mat, _ = sequences_to_matrix(labels, max_seq_len=max_sequence_len)\n",
    "    return input_mat.astype(np.int32), label_mat.astype(np.int32), len_vec.astype(np.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19300\n"
     ]
    }
   ],
   "source": [
    "print(drnn.vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:0 Training NCE Loss = 140.32471\n",
      "0:0 Validation NCE Loss = 118.37694\n",
      "Saving model...\n",
      "10:0 Training NCE Loss = 135.89705\n",
      "20:0 Training NCE Loss = 155.09639\n",
      "30:0 Training NCE Loss = 126.22244\n",
      "40:0 Training NCE Loss = 145.08957\n",
      "50:0 Training NCE Loss = 139.22038\n",
      "60:0 Training NCE Loss = 138.61949\n",
      "70:0 Training NCE Loss = 111.83234\n",
      "80:0 Training NCE Loss = 133.29344\n",
      "90:0 Training NCE Loss = 124.65572\n",
      "100:0 Training NCE Loss = 128.81853\n",
      "100:0 Validation NCE Loss = 118.08834\n",
      "Saving model...\n",
      "110:0 Training NCE Loss = 149.39264\n",
      "120:0 Training NCE Loss = 142.73241\n",
      "130:0 Training NCE Loss = 130.18723\n",
      "140:0 Training NCE Loss = 138.70583\n",
      "0:1 Training NCE Loss = 132.83295\n",
      "10:1 Training NCE Loss = 129.24567\n",
      "20:1 Training NCE Loss = 123.68953\n",
      "30:1 Training NCE Loss = 105.59384\n",
      "40:1 Training NCE Loss = 97.29975\n",
      "50:1 Training NCE Loss = 101.52982\n",
      "60:1 Training NCE Loss = 128.57121\n",
      "70:1 Training NCE Loss = 106.47817\n",
      "80:1 Training NCE Loss = 125.34081\n",
      "90:1 Training NCE Loss = 135.95772\n",
      "100:1 Training NCE Loss = 106.08027\n",
      "110:1 Training NCE Loss = 115.51192\n",
      "120:1 Training NCE Loss = 114.87234\n",
      "130:1 Training NCE Loss = 118.71905\n",
      "140:1 Training NCE Loss = 111.39404\n",
      "0:2 Training NCE Loss = 92.66228\n",
      "10:2 Training NCE Loss = 114.03320\n",
      "20:2 Training NCE Loss = 113.99106\n",
      "30:2 Training NCE Loss = 101.41777\n",
      "40:2 Training NCE Loss = 110.85880\n",
      "50:2 Training NCE Loss = 91.36581\n",
      "60:2 Training NCE Loss = 108.44047\n",
      "70:2 Training NCE Loss = 121.03930\n",
      "80:2 Training NCE Loss = 114.92174\n",
      "90:2 Training NCE Loss = 109.38037\n",
      "100:2 Training NCE Loss = 108.94034\n",
      "110:2 Training NCE Loss = 127.37937\n",
      "120:2 Training NCE Loss = 115.83286\n",
      "130:2 Training NCE Loss = 84.57034\n",
      "140:2 Training NCE Loss = 122.94734\n",
      "0:3 Training NCE Loss = 89.33310\n",
      "10:3 Training NCE Loss = 116.02761\n",
      "20:3 Training NCE Loss = 116.46417\n",
      "30:3 Training NCE Loss = 108.93605\n",
      "40:3 Training NCE Loss = 97.87346\n",
      "50:3 Training NCE Loss = 109.81348\n",
      "60:3 Training NCE Loss = 114.69298\n",
      "70:3 Training NCE Loss = 115.60162\n",
      "80:3 Training NCE Loss = 114.06156\n",
      "90:3 Training NCE Loss = 82.32241\n",
      "100:3 Training NCE Loss = 81.18543\n",
      "110:3 Training NCE Loss = 122.64272\n",
      "120:3 Training NCE Loss = 92.91825\n",
      "130:3 Training NCE Loss = 115.57475\n",
      "140:3 Training NCE Loss = 99.34509\n",
      "0:4 Training NCE Loss = 102.69632\n",
      "10:4 Training NCE Loss = 94.91405\n",
      "20:4 Training NCE Loss = 108.74718\n",
      "30:4 Training NCE Loss = 102.93301\n",
      "40:4 Training NCE Loss = 126.30318\n",
      "50:4 Training NCE Loss = 96.07817\n",
      "60:4 Training NCE Loss = 117.74445\n",
      "70:4 Training NCE Loss = 97.61951\n",
      "80:4 Training NCE Loss = 96.08673\n",
      "90:4 Training NCE Loss = 99.07226\n",
      "100:4 Training NCE Loss = 93.62177\n",
      "110:4 Training NCE Loss = 114.04321\n",
      "120:4 Training NCE Loss = 87.93972\n",
      "130:4 Training NCE Loss = 82.96528\n",
      "140:4 Training NCE Loss = 117.36021\n",
      "0:5 Training NCE Loss = 77.89383\n",
      "10:5 Training NCE Loss = 109.73288\n",
      "20:5 Training NCE Loss = 70.07242\n",
      "30:5 Training NCE Loss = 94.99147\n",
      "40:5 Training NCE Loss = 94.89904\n",
      "50:5 Training NCE Loss = 120.16228\n",
      "60:5 Training NCE Loss = 103.40627\n",
      "70:5 Training NCE Loss = 104.67788\n",
      "80:5 Training NCE Loss = 101.06139\n",
      "90:5 Training NCE Loss = 103.83896\n",
      "100:5 Training NCE Loss = 101.86882\n",
      "110:5 Training NCE Loss = 97.26073\n",
      "120:5 Training NCE Loss = 123.15777\n",
      "130:5 Training NCE Loss = 116.95525\n",
      "140:5 Training NCE Loss = 102.71562\n",
      "0:6 Training NCE Loss = 79.99285\n",
      "10:6 Training NCE Loss = 92.36232\n",
      "20:6 Training NCE Loss = 107.01415\n",
      "30:6 Training NCE Loss = 94.13729\n",
      "40:6 Training NCE Loss = 99.60452\n",
      "50:6 Training NCE Loss = 82.56123\n",
      "60:6 Training NCE Loss = 98.44984\n",
      "70:6 Training NCE Loss = 104.81219\n",
      "80:6 Training NCE Loss = 56.58012\n",
      "90:6 Training NCE Loss = 88.42979\n",
      "100:6 Training NCE Loss = 92.65462\n",
      "110:6 Training NCE Loss = 99.04424\n",
      "120:6 Training NCE Loss = 96.82337\n",
      "130:6 Training NCE Loss = 74.84634\n",
      "140:6 Training NCE Loss = 91.37753\n",
      "0:7 Training NCE Loss = 96.32076\n",
      "10:7 Training NCE Loss = 87.32890\n",
      "20:7 Training NCE Loss = 93.91949\n",
      "30:7 Training NCE Loss = 109.70520\n",
      "40:7 Training NCE Loss = 99.81559\n",
      "50:7 Training NCE Loss = 81.21905\n",
      "60:7 Training NCE Loss = 97.57497\n",
      "70:7 Training NCE Loss = 106.04260\n",
      "80:7 Training NCE Loss = 90.15536\n",
      "90:7 Training NCE Loss = 99.10463\n",
      "100:7 Training NCE Loss = 112.64961\n",
      "110:7 Training NCE Loss = 113.78706\n",
      "120:7 Training NCE Loss = 91.09706\n",
      "130:7 Training NCE Loss = 98.44536\n",
      "140:7 Training NCE Loss = 108.81475\n",
      "0:8 Training NCE Loss = 86.75393\n",
      "10:8 Training NCE Loss = 80.29539\n",
      "20:8 Training NCE Loss = 96.75767\n",
      "30:8 Training NCE Loss = 104.36451\n",
      "40:8 Training NCE Loss = 86.43290\n",
      "50:8 Training NCE Loss = 88.78251\n",
      "60:8 Training NCE Loss = 120.93116\n",
      "70:8 Training NCE Loss = 117.35583\n",
      "80:8 Training NCE Loss = 76.01692\n",
      "90:8 Training NCE Loss = 82.60510\n",
      "100:8 Training NCE Loss = 74.92982\n",
      "110:8 Training NCE Loss = 97.90558\n",
      "120:8 Training NCE Loss = 76.34918\n",
      "130:8 Training NCE Loss = 77.93008\n",
      "140:8 Training NCE Loss = 71.39645\n",
      "0:9 Training NCE Loss = 81.21092\n",
      "10:9 Training NCE Loss = 109.57453\n",
      "20:9 Training NCE Loss = 106.71478\n",
      "30:9 Training NCE Loss = 86.24512\n",
      "40:9 Training NCE Loss = 99.04545\n",
      "50:9 Training NCE Loss = 93.49095\n",
      "60:9 Training NCE Loss = 91.29427\n",
      "70:9 Training NCE Loss = 89.80624\n",
      "80:9 Training NCE Loss = 85.43247\n",
      "90:9 Training NCE Loss = 93.39491\n",
      "100:9 Training NCE Loss = 91.33349\n",
      "110:9 Training NCE Loss = 84.09864\n",
      "120:9 Training NCE Loss = 62.07377\n",
      "130:9 Training NCE Loss = 95.60922\n",
      "140:9 Training NCE Loss = 110.22977\n",
      "0:10 Training NCE Loss = 82.23821\n",
      "10:10 Training NCE Loss = 82.91881\n",
      "20:10 Training NCE Loss = 78.42978\n",
      "30:10 Training NCE Loss = 91.66154\n",
      "40:10 Training NCE Loss = 70.82586\n",
      "50:10 Training NCE Loss = 76.52744\n",
      "60:10 Training NCE Loss = 88.69785\n",
      "70:10 Training NCE Loss = 75.17576\n",
      "80:10 Training NCE Loss = 76.02500\n",
      "90:10 Training NCE Loss = 86.41076\n",
      "100:10 Training NCE Loss = 107.04021\n",
      "110:10 Training NCE Loss = 92.89195\n",
      "120:10 Training NCE Loss = 77.78613\n",
      "130:10 Training NCE Loss = 94.32022\n",
      "140:10 Training NCE Loss = 80.47911\n",
      "0:11 Training NCE Loss = 75.67384\n",
      "10:11 Training NCE Loss = 82.70869\n",
      "20:11 Training NCE Loss = 74.00575\n",
      "30:11 Training NCE Loss = 90.73588\n",
      "40:11 Training NCE Loss = 69.65479\n",
      "50:11 Training NCE Loss = 84.39484\n",
      "60:11 Training NCE Loss = 84.17847\n",
      "70:11 Training NCE Loss = 87.95299\n",
      "80:11 Training NCE Loss = 68.77013\n",
      "90:11 Training NCE Loss = 89.16761\n",
      "100:11 Training NCE Loss = 89.03420\n",
      "110:11 Training NCE Loss = 80.76173\n",
      "120:11 Training NCE Loss = 85.63213\n",
      "130:11 Training NCE Loss = 66.66521\n",
      "140:11 Training NCE Loss = 78.37654\n",
      "0:12 Training NCE Loss = 79.13135\n",
      "10:12 Training NCE Loss = 75.48783\n",
      "20:12 Training NCE Loss = 56.99026\n",
      "30:12 Training NCE Loss = 78.02679\n",
      "40:12 Training NCE Loss = 73.85955\n",
      "50:12 Training NCE Loss = 100.44922\n",
      "60:12 Training NCE Loss = 89.32826\n",
      "70:12 Training NCE Loss = 97.28870\n",
      "80:12 Training NCE Loss = 90.39810\n",
      "90:12 Training NCE Loss = 66.36058\n",
      "100:12 Training NCE Loss = 67.15710\n",
      "110:12 Training NCE Loss = 88.48127\n",
      "120:12 Training NCE Loss = 70.20118\n",
      "130:12 Training NCE Loss = 68.70686\n",
      "140:12 Training NCE Loss = 69.39380\n",
      "0:13 Training NCE Loss = 72.35633\n",
      "10:13 Training NCE Loss = 75.14335\n",
      "20:13 Training NCE Loss = 57.60714\n",
      "30:13 Training NCE Loss = 70.80157\n",
      "40:13 Training NCE Loss = 54.82768\n",
      "50:13 Training NCE Loss = 63.39148\n",
      "60:13 Training NCE Loss = 91.33428\n",
      "70:13 Training NCE Loss = 63.86554\n",
      "80:13 Training NCE Loss = 74.05068\n",
      "90:13 Training NCE Loss = 72.26939\n",
      "100:13 Training NCE Loss = 90.99158\n",
      "110:13 Training NCE Loss = 65.24455\n",
      "120:13 Training NCE Loss = 67.13876\n",
      "130:13 Training NCE Loss = 78.44395\n",
      "140:13 Training NCE Loss = 63.07814\n",
      "0:14 Training NCE Loss = 76.54578\n",
      "10:14 Training NCE Loss = 84.67394\n",
      "20:14 Training NCE Loss = 94.09344\n",
      "30:14 Training NCE Loss = 75.36822\n",
      "40:14 Training NCE Loss = 81.29043\n",
      "50:14 Training NCE Loss = 69.01634\n",
      "60:14 Training NCE Loss = 93.06995\n",
      "70:14 Training NCE Loss = 79.96461\n",
      "80:14 Training NCE Loss = 52.37400\n",
      "90:14 Training NCE Loss = 77.17182\n",
      "100:14 Training NCE Loss = 72.53093\n",
      "110:14 Training NCE Loss = 72.24200\n",
      "120:14 Training NCE Loss = 65.65604\n",
      "130:14 Training NCE Loss = 81.06309\n",
      "140:14 Training NCE Loss = 70.55466\n",
      "0:15 Training NCE Loss = 68.66656\n",
      "10:15 Training NCE Loss = 72.73466\n",
      "20:15 Training NCE Loss = 72.76039\n",
      "30:15 Training NCE Loss = 60.42717\n",
      "40:15 Training NCE Loss = 65.23515\n",
      "50:15 Training NCE Loss = 88.12733\n",
      "60:15 Training NCE Loss = 88.35675\n",
      "70:15 Training NCE Loss = 78.61578\n",
      "80:15 Training NCE Loss = 69.43610\n",
      "90:15 Training NCE Loss = 79.78694\n",
      "100:15 Training NCE Loss = 69.28780\n",
      "110:15 Training NCE Loss = 68.85960\n",
      "120:15 Training NCE Loss = 63.98027\n",
      "130:15 Training NCE Loss = 72.72878\n",
      "140:15 Training NCE Loss = 75.68639\n",
      "0:16 Training NCE Loss = 73.96970\n",
      "10:16 Training NCE Loss = 78.91100\n",
      "20:16 Training NCE Loss = 64.54423\n",
      "30:16 Training NCE Loss = 73.91631\n",
      "40:16 Training NCE Loss = 91.79030\n",
      "50:16 Training NCE Loss = 64.00084\n",
      "60:16 Training NCE Loss = 71.75620\n",
      "70:16 Training NCE Loss = 60.54177\n",
      "80:16 Training NCE Loss = 65.16682\n",
      "90:16 Training NCE Loss = 66.30000\n",
      "100:16 Training NCE Loss = 81.90474\n",
      "110:16 Training NCE Loss = 72.86208\n",
      "120:16 Training NCE Loss = 70.59713\n",
      "130:16 Training NCE Loss = 63.22903\n",
      "140:16 Training NCE Loss = 60.34833\n",
      "0:17 Training NCE Loss = 80.66191\n",
      "10:17 Training NCE Loss = 56.59124\n",
      "20:17 Training NCE Loss = 59.41799\n",
      "30:17 Training NCE Loss = 75.07851\n",
      "40:17 Training NCE Loss = 70.89753\n",
      "50:17 Training NCE Loss = 78.47500\n",
      "60:17 Training NCE Loss = 76.42604\n",
      "70:17 Training NCE Loss = 91.09721\n",
      "80:17 Training NCE Loss = 63.87737\n",
      "90:17 Training NCE Loss = 67.62740\n",
      "100:17 Training NCE Loss = 79.65882\n",
      "110:17 Training NCE Loss = 62.79925\n",
      "120:17 Training NCE Loss = 95.63067\n",
      "130:17 Training NCE Loss = 50.82890\n",
      "140:17 Training NCE Loss = 61.29971\n",
      "0:18 Training NCE Loss = 80.91214\n",
      "10:18 Training NCE Loss = 64.12342\n",
      "20:18 Training NCE Loss = 73.10818\n",
      "30:18 Training NCE Loss = 61.63628\n",
      "40:18 Training NCE Loss = 55.09571\n",
      "50:18 Training NCE Loss = 73.50556\n",
      "60:18 Training NCE Loss = 59.88590\n",
      "70:18 Training NCE Loss = 69.08939\n",
      "80:18 Training NCE Loss = 62.73944\n",
      "90:18 Training NCE Loss = 57.27113\n",
      "100:18 Training NCE Loss = 68.09735\n",
      "110:18 Training NCE Loss = 69.14692\n",
      "120:18 Training NCE Loss = 66.73297\n",
      "130:18 Training NCE Loss = 67.34204\n",
      "140:18 Training NCE Loss = 76.83389\n",
      "0:19 Training NCE Loss = 67.65392\n",
      "10:19 Training NCE Loss = 69.68986\n",
      "20:19 Training NCE Loss = 58.74194\n",
      "30:19 Training NCE Loss = 97.10824\n",
      "40:19 Training NCE Loss = 62.94443\n",
      "50:19 Training NCE Loss = 58.48693\n",
      "60:19 Training NCE Loss = 69.57612\n",
      "70:19 Training NCE Loss = 67.64210\n",
      "80:19 Training NCE Loss = 103.82325\n",
      "90:19 Training NCE Loss = 63.71050\n",
      "100:19 Training NCE Loss = 64.97198\n",
      "110:19 Training NCE Loss = 89.32009\n",
      "120:19 Training NCE Loss = 52.72408\n",
      "130:19 Training NCE Loss = 52.63210\n",
      "140:19 Training NCE Loss = 80.98203\n",
      "0:20 Training NCE Loss = 71.91387\n",
      "10:20 Training NCE Loss = 66.27062\n",
      "20:20 Training NCE Loss = 89.65433\n",
      "30:20 Training NCE Loss = 48.38430\n",
      "40:20 Training NCE Loss = 89.69482\n",
      "50:20 Training NCE Loss = 82.39639\n",
      "60:20 Training NCE Loss = 63.09315\n",
      "70:20 Training NCE Loss = 53.73001\n",
      "80:20 Training NCE Loss = 66.32883\n",
      "90:20 Training NCE Loss = 48.09365\n",
      "100:20 Training NCE Loss = 40.39655\n",
      "110:20 Training NCE Loss = 70.80908\n",
      "120:20 Training NCE Loss = 46.77964\n",
      "130:20 Training NCE Loss = 87.56720\n",
      "140:20 Training NCE Loss = 46.58280\n",
      "0:21 Training NCE Loss = 76.22030\n",
      "10:21 Training NCE Loss = 72.08736\n",
      "20:21 Training NCE Loss = 64.62827\n",
      "30:21 Training NCE Loss = 51.26320\n",
      "40:21 Training NCE Loss = 47.33315\n",
      "50:21 Training NCE Loss = 43.23523\n",
      "60:21 Training NCE Loss = 53.86726\n",
      "70:21 Training NCE Loss = 81.35397\n",
      "80:21 Training NCE Loss = 91.90491\n",
      "90:21 Training NCE Loss = 59.90882\n",
      "100:21 Training NCE Loss = 69.71017\n",
      "110:21 Training NCE Loss = 61.40137\n",
      "120:21 Training NCE Loss = 51.20835\n",
      "130:21 Training NCE Loss = 56.41932\n",
      "140:21 Training NCE Loss = 53.89019\n",
      "0:22 Training NCE Loss = 52.02505\n",
      "10:22 Training NCE Loss = 53.23731\n",
      "20:22 Training NCE Loss = 61.05803\n",
      "30:22 Training NCE Loss = 55.11795\n",
      "40:22 Training NCE Loss = 62.60435\n",
      "50:22 Training NCE Loss = 78.19138\n",
      "60:22 Training NCE Loss = 44.29350\n",
      "70:22 Training NCE Loss = 62.07107\n",
      "80:22 Training NCE Loss = 63.15478\n",
      "90:22 Training NCE Loss = 74.45680\n",
      "100:22 Training NCE Loss = 51.58506\n",
      "110:22 Training NCE Loss = 72.71786\n",
      "120:22 Training NCE Loss = 71.72629\n",
      "130:22 Training NCE Loss = 55.10849\n",
      "140:22 Training NCE Loss = 52.65387\n",
      "0:23 Training NCE Loss = 80.74366\n",
      "10:23 Training NCE Loss = 58.02466\n",
      "20:23 Training NCE Loss = 63.42685\n",
      "30:23 Training NCE Loss = 95.05984\n",
      "40:23 Training NCE Loss = 80.17745\n",
      "50:23 Training NCE Loss = 65.32514\n",
      "60:23 Training NCE Loss = 52.63435\n",
      "70:23 Training NCE Loss = 57.96430\n",
      "80:23 Training NCE Loss = 60.30515\n",
      "90:23 Training NCE Loss = 32.76386\n",
      "100:23 Training NCE Loss = 72.91965\n",
      "110:23 Training NCE Loss = 66.68678\n",
      "120:23 Training NCE Loss = 44.23286\n",
      "130:23 Training NCE Loss = 72.22942\n",
      "140:23 Training NCE Loss = 62.27691\n",
      "0:24 Training NCE Loss = 53.66726\n",
      "10:24 Training NCE Loss = 30.48379\n",
      "20:24 Training NCE Loss = 63.64466\n",
      "30:24 Training NCE Loss = 62.58377\n",
      "40:24 Training NCE Loss = 61.13890\n",
      "50:24 Training NCE Loss = 66.58997\n",
      "60:24 Training NCE Loss = 49.63050\n",
      "70:24 Training NCE Loss = 43.06031\n",
      "80:24 Training NCE Loss = 37.25998\n",
      "90:24 Training NCE Loss = 74.20779\n",
      "100:24 Training NCE Loss = 61.31419\n",
      "110:24 Training NCE Loss = 42.28760\n",
      "120:24 Training NCE Loss = 60.12904\n",
      "130:24 Training NCE Loss = 74.28427\n",
      "140:24 Training NCE Loss = 64.08640\n",
      "0:25 Training NCE Loss = 73.13522\n",
      "10:25 Training NCE Loss = 62.64210\n",
      "20:25 Training NCE Loss = 78.87007\n",
      "30:25 Training NCE Loss = 64.36217\n",
      "40:25 Training NCE Loss = 74.26712\n",
      "50:25 Training NCE Loss = 55.79914\n",
      "60:25 Training NCE Loss = 59.57839\n",
      "70:25 Training NCE Loss = 58.55090\n",
      "80:25 Training NCE Loss = 50.88144\n",
      "90:25 Training NCE Loss = 57.21236\n",
      "100:25 Training NCE Loss = 60.78425\n",
      "110:25 Training NCE Loss = 63.77293\n",
      "120:25 Training NCE Loss = 54.44101\n",
      "130:25 Training NCE Loss = 60.79778\n",
      "140:25 Training NCE Loss = 67.87229\n",
      "0:26 Training NCE Loss = 48.93272\n",
      "10:26 Training NCE Loss = 72.17433\n",
      "20:26 Training NCE Loss = 78.31873\n",
      "30:26 Training NCE Loss = 43.57000\n",
      "40:26 Training NCE Loss = 65.16818\n",
      "50:26 Training NCE Loss = 53.77321\n",
      "60:26 Training NCE Loss = 58.22387\n",
      "70:26 Training NCE Loss = 59.30957\n",
      "80:26 Training NCE Loss = 65.32339\n",
      "90:26 Training NCE Loss = 67.41460\n",
      "100:26 Training NCE Loss = 63.54808\n",
      "110:26 Training NCE Loss = 72.17344\n",
      "120:26 Training NCE Loss = 68.73776\n",
      "130:26 Training NCE Loss = 68.04361\n",
      "140:26 Training NCE Loss = 59.00243\n",
      "0:27 Training NCE Loss = 62.77627\n",
      "10:27 Training NCE Loss = 49.92894\n",
      "20:27 Training NCE Loss = 63.03843\n",
      "30:27 Training NCE Loss = 52.25153\n",
      "40:27 Training NCE Loss = 49.07744\n",
      "50:27 Training NCE Loss = 55.59719\n",
      "60:27 Training NCE Loss = 66.97481\n",
      "70:27 Training NCE Loss = 58.76402\n",
      "80:27 Training NCE Loss = 47.65978\n",
      "90:27 Training NCE Loss = 46.23649\n",
      "100:27 Training NCE Loss = 57.92470\n",
      "110:27 Training NCE Loss = 60.96495\n",
      "120:27 Training NCE Loss = 55.30353\n",
      "130:27 Training NCE Loss = 60.70467\n",
      "140:27 Training NCE Loss = 39.27634\n",
      "0:28 Training NCE Loss = 80.55679\n",
      "10:28 Training NCE Loss = 48.97902\n",
      "20:28 Training NCE Loss = 62.72466\n",
      "30:28 Training NCE Loss = 64.23951\n",
      "40:28 Training NCE Loss = 66.83775\n",
      "50:28 Training NCE Loss = 48.94886\n",
      "60:28 Training NCE Loss = 61.17530\n",
      "70:28 Training NCE Loss = 54.74128\n",
      "80:28 Training NCE Loss = 43.03202\n",
      "90:28 Training NCE Loss = 51.54253\n",
      "100:28 Training NCE Loss = 59.39888\n",
      "110:28 Training NCE Loss = 71.25641\n",
      "120:28 Training NCE Loss = 72.29115\n",
      "130:28 Training NCE Loss = 40.10678\n",
      "140:28 Training NCE Loss = 50.42570\n",
      "0:29 Training NCE Loss = 78.66616\n",
      "10:29 Training NCE Loss = 60.74424\n",
      "20:29 Training NCE Loss = 56.04495\n",
      "30:29 Training NCE Loss = 58.59771\n",
      "40:29 Training NCE Loss = 61.01196\n",
      "50:29 Training NCE Loss = 50.12007\n",
      "60:29 Training NCE Loss = 53.49551\n",
      "70:29 Training NCE Loss = 56.63272\n",
      "80:29 Training NCE Loss = 51.91120\n",
      "90:29 Training NCE Loss = 76.06206\n",
      "100:29 Training NCE Loss = 66.57638\n",
      "110:29 Training NCE Loss = 40.35108\n",
      "120:29 Training NCE Loss = 56.39752\n",
      "130:29 Training NCE Loss = 35.07307\n",
      "140:29 Training NCE Loss = 58.46334\n",
      "0:30 Training NCE Loss = 53.08878\n",
      "10:30 Training NCE Loss = 45.53598\n",
      "20:30 Training NCE Loss = 49.93441\n",
      "30:30 Training NCE Loss = 45.31868\n",
      "40:30 Training NCE Loss = 52.77288\n",
      "50:30 Training NCE Loss = 44.89884\n",
      "60:30 Training NCE Loss = 64.50123\n",
      "70:30 Training NCE Loss = 43.04090\n",
      "80:30 Training NCE Loss = 60.99100\n",
      "90:30 Training NCE Loss = 57.45866\n",
      "100:30 Training NCE Loss = 61.45359\n",
      "110:30 Training NCE Loss = 55.72373\n",
      "120:30 Training NCE Loss = 48.78376\n",
      "130:30 Training NCE Loss = 53.77965\n",
      "140:30 Training NCE Loss = 62.33918\n",
      "0:31 Training NCE Loss = 80.84153\n",
      "10:31 Training NCE Loss = 27.17735\n",
      "20:31 Training NCE Loss = 47.46921\n",
      "30:31 Training NCE Loss = 69.39870\n",
      "40:31 Training NCE Loss = 37.53498\n",
      "50:31 Training NCE Loss = 55.30552\n",
      "60:31 Training NCE Loss = 43.81222\n",
      "70:31 Training NCE Loss = 82.34816\n",
      "80:31 Training NCE Loss = 50.88816\n",
      "90:31 Training NCE Loss = 70.21187\n",
      "100:31 Training NCE Loss = 46.08331\n",
      "110:31 Training NCE Loss = 69.21574\n",
      "120:31 Training NCE Loss = 46.47237\n",
      "130:31 Training NCE Loss = 50.71304\n",
      "140:31 Training NCE Loss = 80.11993\n",
      "0:32 Training NCE Loss = 49.15820\n",
      "10:32 Training NCE Loss = 42.24358\n",
      "20:32 Training NCE Loss = 43.73386\n",
      "30:32 Training NCE Loss = 49.30321\n",
      "40:32 Training NCE Loss = 58.12932\n",
      "50:32 Training NCE Loss = 59.04824\n",
      "60:32 Training NCE Loss = 59.71085\n",
      "70:32 Training NCE Loss = 43.80722\n",
      "80:32 Training NCE Loss = 58.56590\n",
      "90:32 Training NCE Loss = 36.16481\n",
      "100:32 Training NCE Loss = 62.21882\n",
      "110:32 Training NCE Loss = 68.88031\n",
      "120:32 Training NCE Loss = 47.62406\n",
      "130:32 Training NCE Loss = 51.96534\n",
      "140:32 Training NCE Loss = 62.21501\n",
      "0:33 Training NCE Loss = 49.95845\n",
      "10:33 Training NCE Loss = 43.89423\n",
      "20:33 Training NCE Loss = 67.91843\n",
      "30:33 Training NCE Loss = 48.54672\n",
      "40:33 Training NCE Loss = 62.49695\n",
      "50:33 Training NCE Loss = 38.73408\n",
      "60:33 Training NCE Loss = 47.39379\n",
      "70:33 Training NCE Loss = 60.69278\n",
      "80:33 Training NCE Loss = 46.24194\n",
      "90:33 Training NCE Loss = 49.77457\n",
      "100:33 Training NCE Loss = 43.88218\n",
      "110:33 Training NCE Loss = 43.94693\n",
      "120:33 Training NCE Loss = 52.19515\n",
      "130:33 Training NCE Loss = 47.81229\n",
      "140:33 Training NCE Loss = 47.26098\n",
      "0:34 Training NCE Loss = 47.16859\n",
      "10:34 Training NCE Loss = 50.54765\n",
      "20:34 Training NCE Loss = 39.67994\n",
      "30:34 Training NCE Loss = 51.46321\n",
      "40:34 Training NCE Loss = 36.37945\n",
      "50:34 Training NCE Loss = 84.03288\n",
      "60:34 Training NCE Loss = 44.37990\n",
      "70:34 Training NCE Loss = 61.40526\n",
      "80:34 Training NCE Loss = 59.25408\n",
      "90:34 Training NCE Loss = 41.94737\n",
      "100:34 Training NCE Loss = 41.12974\n",
      "110:34 Training NCE Loss = 31.03089\n",
      "120:34 Training NCE Loss = 24.91411\n",
      "130:34 Training NCE Loss = 48.14788\n",
      "140:34 Training NCE Loss = 24.10763\n",
      "0:35 Training NCE Loss = 61.37334\n",
      "10:35 Training NCE Loss = 57.30930\n",
      "20:35 Training NCE Loss = 45.78152\n",
      "30:35 Training NCE Loss = 54.54295\n",
      "40:35 Training NCE Loss = 51.38858\n",
      "50:35 Training NCE Loss = 61.90080\n",
      "60:35 Training NCE Loss = 67.46703\n",
      "70:35 Training NCE Loss = 57.93727\n",
      "80:35 Training NCE Loss = 46.99001\n",
      "90:35 Training NCE Loss = 63.82528\n",
      "100:35 Training NCE Loss = 74.09019\n",
      "110:35 Training NCE Loss = 50.95675\n",
      "120:35 Training NCE Loss = 41.34933\n",
      "130:35 Training NCE Loss = 66.61887\n",
      "140:35 Training NCE Loss = 65.32577\n",
      "0:36 Training NCE Loss = 51.89083\n",
      "10:36 Training NCE Loss = 34.34039\n",
      "20:36 Training NCE Loss = 33.18753\n",
      "30:36 Training NCE Loss = 53.77967\n",
      "40:36 Training NCE Loss = 34.62725\n",
      "50:36 Training NCE Loss = 52.12532\n",
      "60:36 Training NCE Loss = 48.30387\n",
      "70:36 Training NCE Loss = 59.20654\n",
      "80:36 Training NCE Loss = 33.26573\n",
      "90:36 Training NCE Loss = 36.28277\n",
      "100:36 Training NCE Loss = 61.74295\n",
      "110:36 Training NCE Loss = 58.11546\n",
      "120:36 Training NCE Loss = 47.69568\n",
      "130:36 Training NCE Loss = 46.88483\n",
      "140:36 Training NCE Loss = 50.04003\n",
      "0:37 Training NCE Loss = 33.73167\n",
      "10:37 Training NCE Loss = 45.36709\n",
      "20:37 Training NCE Loss = 45.76015\n",
      "30:37 Training NCE Loss = 52.96626\n",
      "40:37 Training NCE Loss = 59.84113\n",
      "50:37 Training NCE Loss = 34.34281\n",
      "60:37 Training NCE Loss = 37.95526\n",
      "70:37 Training NCE Loss = 36.40460\n",
      "80:37 Training NCE Loss = 46.29277\n",
      "90:37 Training NCE Loss = 51.16837\n",
      "100:37 Training NCE Loss = 39.91648\n",
      "110:37 Training NCE Loss = 55.56469\n",
      "120:37 Training NCE Loss = 66.10598\n",
      "130:37 Training NCE Loss = 47.21347\n",
      "140:37 Training NCE Loss = 67.11423\n",
      "0:38 Training NCE Loss = 58.01012\n",
      "10:38 Training NCE Loss = 55.06369\n",
      "20:38 Training NCE Loss = 48.17054\n",
      "30:38 Training NCE Loss = 46.92826\n",
      "40:38 Training NCE Loss = 45.63467\n",
      "50:38 Training NCE Loss = 54.78743\n",
      "60:38 Training NCE Loss = 50.82777\n",
      "70:38 Training NCE Loss = 47.52958\n",
      "80:38 Training NCE Loss = 69.44923\n",
      "90:38 Training NCE Loss = 59.07580\n",
      "100:38 Training NCE Loss = 50.57262\n",
      "110:38 Training NCE Loss = 61.65474\n",
      "120:38 Training NCE Loss = 46.44646\n",
      "130:38 Training NCE Loss = 54.79140\n",
      "140:38 Training NCE Loss = 55.24802\n",
      "0:39 Training NCE Loss = 42.07229\n",
      "10:39 Training NCE Loss = 48.05970\n",
      "20:39 Training NCE Loss = 38.67993\n",
      "30:39 Training NCE Loss = 44.52698\n",
      "40:39 Training NCE Loss = 31.53029\n",
      "50:39 Training NCE Loss = 63.19226\n",
      "60:39 Training NCE Loss = 42.16774\n",
      "70:39 Training NCE Loss = 50.17237\n",
      "80:39 Training NCE Loss = 52.23446\n",
      "90:39 Training NCE Loss = 58.68286\n",
      "100:39 Training NCE Loss = 53.80339\n",
      "110:39 Training NCE Loss = 53.12748\n",
      "120:39 Training NCE Loss = 54.80358\n",
      "130:39 Training NCE Loss = 46.15963\n",
      "140:39 Training NCE Loss = 43.85630\n",
      "0:40 Training NCE Loss = 56.24848\n",
      "10:40 Training NCE Loss = 52.69507\n",
      "20:40 Training NCE Loss = 42.15204\n",
      "30:40 Training NCE Loss = 44.73061\n",
      "40:40 Training NCE Loss = 26.17742\n",
      "50:40 Training NCE Loss = 32.20186\n",
      "60:40 Training NCE Loss = 58.20353\n",
      "70:40 Training NCE Loss = 53.43832\n",
      "80:40 Training NCE Loss = 60.28490\n",
      "90:40 Training NCE Loss = 53.00169\n",
      "100:40 Training NCE Loss = 48.80189\n",
      "110:40 Training NCE Loss = 43.38726\n",
      "120:40 Training NCE Loss = 39.82882\n",
      "130:40 Training NCE Loss = 52.25673\n",
      "140:40 Training NCE Loss = 37.52746\n",
      "0:41 Training NCE Loss = 36.11315\n",
      "10:41 Training NCE Loss = 50.84444\n",
      "20:41 Training NCE Loss = 50.12271\n",
      "30:41 Training NCE Loss = 51.24502\n",
      "40:41 Training NCE Loss = 41.43147\n",
      "50:41 Training NCE Loss = 48.37244\n",
      "60:41 Training NCE Loss = 35.00211\n",
      "70:41 Training NCE Loss = 67.57233\n",
      "80:41 Training NCE Loss = 33.72348\n",
      "90:41 Training NCE Loss = 44.70575\n",
      "100:41 Training NCE Loss = 47.28106\n",
      "110:41 Training NCE Loss = 49.21615\n",
      "120:41 Training NCE Loss = 44.99997\n",
      "130:41 Training NCE Loss = 62.35664\n",
      "140:41 Training NCE Loss = 39.93871\n",
      "0:42 Training NCE Loss = 52.22394\n",
      "10:42 Training NCE Loss = 66.35053\n",
      "20:42 Training NCE Loss = 36.17964\n",
      "30:42 Training NCE Loss = 40.53925\n",
      "40:42 Training NCE Loss = 36.14166\n",
      "50:42 Training NCE Loss = 49.23611\n",
      "60:42 Training NCE Loss = 42.29854\n",
      "70:42 Training NCE Loss = 39.11268\n",
      "80:42 Training NCE Loss = 43.88230\n",
      "90:42 Training NCE Loss = 34.17873\n",
      "100:42 Training NCE Loss = 51.08887\n",
      "110:42 Training NCE Loss = 55.92396\n",
      "120:42 Training NCE Loss = 42.13093\n",
      "130:42 Training NCE Loss = 25.32033\n",
      "140:42 Training NCE Loss = 34.15959\n",
      "0:43 Training NCE Loss = 55.73317\n",
      "10:43 Training NCE Loss = 45.62536\n",
      "20:43 Training NCE Loss = 38.33788\n",
      "30:43 Training NCE Loss = 41.22894\n",
      "40:43 Training NCE Loss = 36.26793\n",
      "50:43 Training NCE Loss = 33.58902\n",
      "60:43 Training NCE Loss = 48.07329\n",
      "70:43 Training NCE Loss = 48.68737\n",
      "80:43 Training NCE Loss = 35.02467\n",
      "90:43 Training NCE Loss = 56.92856\n",
      "100:43 Training NCE Loss = 44.19965\n",
      "110:43 Training NCE Loss = 29.58295\n",
      "120:43 Training NCE Loss = 44.60498\n",
      "130:43 Training NCE Loss = 40.54675\n",
      "140:43 Training NCE Loss = 53.24014\n",
      "0:44 Training NCE Loss = 42.45452\n",
      "10:44 Training NCE Loss = 52.59878\n",
      "20:44 Training NCE Loss = 48.03527\n",
      "30:44 Training NCE Loss = 27.30791\n",
      "40:44 Training NCE Loss = 28.57808\n",
      "50:44 Training NCE Loss = 33.08547\n",
      "60:44 Training NCE Loss = 48.50359\n",
      "70:44 Training NCE Loss = 49.60323\n",
      "80:44 Training NCE Loss = 36.04457\n",
      "90:44 Training NCE Loss = 32.50461\n",
      "100:44 Training NCE Loss = 36.67766\n",
      "110:44 Training NCE Loss = 42.45697\n",
      "120:44 Training NCE Loss = 43.64756\n",
      "130:44 Training NCE Loss = 57.75673\n",
      "140:44 Training NCE Loss = 46.13326\n",
      "0:45 Training NCE Loss = 42.78078\n",
      "10:45 Training NCE Loss = 46.90008\n",
      "20:45 Training NCE Loss = 64.20707\n",
      "30:45 Training NCE Loss = 43.04707\n",
      "40:45 Training NCE Loss = 63.04748\n",
      "50:45 Training NCE Loss = 56.40369\n",
      "60:45 Training NCE Loss = 47.70886\n",
      "70:45 Training NCE Loss = 58.27028\n",
      "80:45 Training NCE Loss = 41.17071\n",
      "90:45 Training NCE Loss = 22.44055\n",
      "100:45 Training NCE Loss = 45.26521\n",
      "110:45 Training NCE Loss = 51.38396\n",
      "120:45 Training NCE Loss = 36.43179\n",
      "130:45 Training NCE Loss = 58.45179\n",
      "140:45 Training NCE Loss = 42.12032\n",
      "0:46 Training NCE Loss = 39.16724\n",
      "10:46 Training NCE Loss = 58.08725\n",
      "20:46 Training NCE Loss = 34.84363\n",
      "30:46 Training NCE Loss = 47.12407\n",
      "40:46 Training NCE Loss = 37.50314\n",
      "50:46 Training NCE Loss = 28.98138\n",
      "60:46 Training NCE Loss = 42.39117\n",
      "70:46 Training NCE Loss = 40.01284\n",
      "80:46 Training NCE Loss = 59.99792\n",
      "90:46 Training NCE Loss = 36.52437\n",
      "100:46 Training NCE Loss = 31.55510\n",
      "110:46 Training NCE Loss = 64.85506\n",
      "120:46 Training NCE Loss = 42.43812\n",
      "130:46 Training NCE Loss = 36.53732\n",
      "140:46 Training NCE Loss = 40.85152\n",
      "0:47 Training NCE Loss = 40.09322\n",
      "10:47 Training NCE Loss = 30.26025\n",
      "20:47 Training NCE Loss = 49.81112\n",
      "30:47 Training NCE Loss = 42.59755\n",
      "40:47 Training NCE Loss = 38.26638\n",
      "50:47 Training NCE Loss = 47.65943\n",
      "60:47 Training NCE Loss = 60.18880\n",
      "70:47 Training NCE Loss = 39.99633\n",
      "80:47 Training NCE Loss = 44.03657\n",
      "90:47 Training NCE Loss = 44.22073\n",
      "100:47 Training NCE Loss = 53.70256\n",
      "110:47 Training NCE Loss = 35.15126\n",
      "120:47 Training NCE Loss = 41.18991\n",
      "130:47 Training NCE Loss = 21.48279\n",
      "140:47 Training NCE Loss = 34.99944\n",
      "0:48 Training NCE Loss = 26.82438\n",
      "10:48 Training NCE Loss = 37.95963\n",
      "20:48 Training NCE Loss = 36.17093\n",
      "30:48 Training NCE Loss = 43.15615\n",
      "40:48 Training NCE Loss = 37.59798\n",
      "50:48 Training NCE Loss = 44.91927\n",
      "60:48 Training NCE Loss = 41.67894\n",
      "70:48 Training NCE Loss = 46.98849\n",
      "80:48 Training NCE Loss = 45.60622\n",
      "90:48 Training NCE Loss = 36.23635\n",
      "100:48 Training NCE Loss = 34.79073\n",
      "110:48 Training NCE Loss = 36.19522\n",
      "120:48 Training NCE Loss = 32.53542\n",
      "130:48 Training NCE Loss = 74.93488\n",
      "140:48 Training NCE Loss = 24.29530\n",
      "0:49 Training NCE Loss = 43.90884\n",
      "10:49 Training NCE Loss = 42.93410\n",
      "20:49 Training NCE Loss = 30.06948\n",
      "30:49 Training NCE Loss = 52.95461\n",
      "40:49 Training NCE Loss = 61.35142\n",
      "50:49 Training NCE Loss = 31.11250\n",
      "60:49 Training NCE Loss = 28.12638\n",
      "70:49 Training NCE Loss = 36.40440\n",
      "80:49 Training NCE Loss = 65.78918\n",
      "90:49 Training NCE Loss = 36.45382\n",
      "100:49 Training NCE Loss = 38.00568\n",
      "110:49 Training NCE Loss = 42.75978\n",
      "120:49 Training NCE Loss = 53.97239\n",
      "130:49 Training NCE Loss = 31.99342\n",
      "140:49 Training NCE Loss = 44.39335\n",
      "0:50 Training NCE Loss = 44.63055\n",
      "10:50 Training NCE Loss = 57.06368\n",
      "20:50 Training NCE Loss = 50.58937\n",
      "30:50 Training NCE Loss = 44.02118\n",
      "40:50 Training NCE Loss = 31.13716\n",
      "50:50 Training NCE Loss = 62.91781\n",
      "60:50 Training NCE Loss = 43.43776\n",
      "70:50 Training NCE Loss = 35.76656\n",
      "80:50 Training NCE Loss = 40.30175\n",
      "90:50 Training NCE Loss = 40.54545\n",
      "100:50 Training NCE Loss = 53.29950\n",
      "110:50 Training NCE Loss = 43.51550\n",
      "120:50 Training NCE Loss = 40.59342\n",
      "130:50 Training NCE Loss = 44.79608\n",
      "140:50 Training NCE Loss = 25.50162\n",
      "0:51 Training NCE Loss = 42.75128\n",
      "10:51 Training NCE Loss = 45.33382\n",
      "20:51 Training NCE Loss = 44.16343\n",
      "30:51 Training NCE Loss = 21.20197\n",
      "40:51 Training NCE Loss = 41.41403\n",
      "50:51 Training NCE Loss = 35.07030\n",
      "60:51 Training NCE Loss = 44.99952\n",
      "70:51 Training NCE Loss = 29.42019\n",
      "80:51 Training NCE Loss = 29.47095\n",
      "90:51 Training NCE Loss = 41.89851\n",
      "100:51 Training NCE Loss = 33.72547\n",
      "110:51 Training NCE Loss = 42.22212\n",
      "120:51 Training NCE Loss = 45.02458\n",
      "130:51 Training NCE Loss = 29.91729\n",
      "140:51 Training NCE Loss = 18.07307\n",
      "0:52 Training NCE Loss = 31.19535\n",
      "10:52 Training NCE Loss = 60.28747\n",
      "20:52 Training NCE Loss = 24.05466\n",
      "30:52 Training NCE Loss = 39.74520\n",
      "40:52 Training NCE Loss = 36.27220\n",
      "50:52 Training NCE Loss = 52.49936\n",
      "60:52 Training NCE Loss = 25.81216\n",
      "70:52 Training NCE Loss = 47.82573\n",
      "80:52 Training NCE Loss = 32.69913\n",
      "90:52 Training NCE Loss = 37.05167\n",
      "100:52 Training NCE Loss = 38.89499\n",
      "110:52 Training NCE Loss = 47.76940\n",
      "120:52 Training NCE Loss = 43.71308\n",
      "130:52 Training NCE Loss = 38.90728\n",
      "140:52 Training NCE Loss = 20.98454\n",
      "0:53 Training NCE Loss = 27.72657\n",
      "10:53 Training NCE Loss = 34.91707\n",
      "20:53 Training NCE Loss = 29.69241\n",
      "30:53 Training NCE Loss = 45.01532\n",
      "40:53 Training NCE Loss = 51.80916\n",
      "50:53 Training NCE Loss = 38.25570\n",
      "60:53 Training NCE Loss = 51.59039\n",
      "70:53 Training NCE Loss = 49.08743\n",
      "80:53 Training NCE Loss = 46.73831\n",
      "90:53 Training NCE Loss = 44.93411\n",
      "100:53 Training NCE Loss = 28.92756\n",
      "110:53 Training NCE Loss = 29.75407\n",
      "120:53 Training NCE Loss = 35.88367\n",
      "130:53 Training NCE Loss = 26.33688\n",
      "140:53 Training NCE Loss = 37.70334\n",
      "0:54 Training NCE Loss = 32.74930\n",
      "10:54 Training NCE Loss = 48.57602\n",
      "20:54 Training NCE Loss = 44.74926\n",
      "30:54 Training NCE Loss = 37.69558\n",
      "40:54 Training NCE Loss = 52.74038\n",
      "50:54 Training NCE Loss = 28.65119\n",
      "60:54 Training NCE Loss = 44.15089\n",
      "70:54 Training NCE Loss = 28.95360\n",
      "80:54 Training NCE Loss = 42.12459\n",
      "90:54 Training NCE Loss = 26.93893\n",
      "100:54 Training NCE Loss = 42.64522\n",
      "110:54 Training NCE Loss = 31.69117\n",
      "120:54 Training NCE Loss = 29.25904\n",
      "130:54 Training NCE Loss = 51.35251\n",
      "140:54 Training NCE Loss = 33.49630\n",
      "0:55 Training NCE Loss = 44.03187\n",
      "10:55 Training NCE Loss = 50.04813\n",
      "20:55 Training NCE Loss = 34.60970\n",
      "30:55 Training NCE Loss = 24.96730\n",
      "40:55 Training NCE Loss = 27.60964\n",
      "50:55 Training NCE Loss = 32.08604\n",
      "60:55 Training NCE Loss = 52.15004\n",
      "70:55 Training NCE Loss = 51.89310\n",
      "80:55 Training NCE Loss = 46.96264\n",
      "90:55 Training NCE Loss = 44.23717\n",
      "100:55 Training NCE Loss = 29.06181\n",
      "110:55 Training NCE Loss = 23.39835\n",
      "120:55 Training NCE Loss = 30.71549\n",
      "130:55 Training NCE Loss = 20.25030\n",
      "140:55 Training NCE Loss = 24.20444\n",
      "0:56 Training NCE Loss = 45.83182\n",
      "10:56 Training NCE Loss = 33.31597\n",
      "20:56 Training NCE Loss = 23.86407\n",
      "30:56 Training NCE Loss = 28.66573\n",
      "40:56 Training NCE Loss = 24.28787\n",
      "50:56 Training NCE Loss = 34.26660\n",
      "60:56 Training NCE Loss = 51.52620\n",
      "70:56 Training NCE Loss = 25.69910\n",
      "80:56 Training NCE Loss = 34.29157\n",
      "90:56 Training NCE Loss = 38.10553\n",
      "100:56 Training NCE Loss = 25.30045\n",
      "110:56 Training NCE Loss = 34.48651\n",
      "120:56 Training NCE Loss = 40.88462\n",
      "130:56 Training NCE Loss = 24.21371\n",
      "140:56 Training NCE Loss = 42.31684\n",
      "0:57 Training NCE Loss = 27.31769\n",
      "10:57 Training NCE Loss = 54.37022\n",
      "20:57 Training NCE Loss = 27.03308\n",
      "30:57 Training NCE Loss = 32.80225\n",
      "40:57 Training NCE Loss = 40.52081\n",
      "50:57 Training NCE Loss = 24.82862\n",
      "60:57 Training NCE Loss = 49.11952\n",
      "70:57 Training NCE Loss = 40.74335\n",
      "80:57 Training NCE Loss = 19.04725\n",
      "90:57 Training NCE Loss = 53.46791\n",
      "100:57 Training NCE Loss = 44.21347\n",
      "110:57 Training NCE Loss = 34.98351\n",
      "120:57 Training NCE Loss = 25.34173\n",
      "130:57 Training NCE Loss = 43.91471\n",
      "140:57 Training NCE Loss = 50.30664\n",
      "0:58 Training NCE Loss = 39.07019\n",
      "10:58 Training NCE Loss = 53.57347\n",
      "20:58 Training NCE Loss = 37.93956\n",
      "30:58 Training NCE Loss = 43.78966\n",
      "40:58 Training NCE Loss = 39.63565\n",
      "50:58 Training NCE Loss = 32.76472\n",
      "60:58 Training NCE Loss = 35.03191\n",
      "70:58 Training NCE Loss = 34.16555\n",
      "80:58 Training NCE Loss = 30.02334\n",
      "90:58 Training NCE Loss = 27.94097\n",
      "100:58 Training NCE Loss = 49.86418\n",
      "110:58 Training NCE Loss = 42.74020\n",
      "120:58 Training NCE Loss = 32.49826\n",
      "130:58 Training NCE Loss = 52.09733\n",
      "140:58 Training NCE Loss = 35.64944\n",
      "0:59 Training NCE Loss = 51.07817\n",
      "10:59 Training NCE Loss = 25.26879\n",
      "20:59 Training NCE Loss = 55.83958\n",
      "30:59 Training NCE Loss = 37.16082\n",
      "40:59 Training NCE Loss = 53.42640\n",
      "50:59 Training NCE Loss = 30.27634\n",
      "60:59 Training NCE Loss = 35.17865\n",
      "70:59 Training NCE Loss = 40.14522\n",
      "80:59 Training NCE Loss = 42.80472\n",
      "90:59 Training NCE Loss = 15.82935\n",
      "100:59 Training NCE Loss = 41.38458\n",
      "110:59 Training NCE Loss = 30.71469\n",
      "120:59 Training NCE Loss = 49.58620\n",
      "130:59 Training NCE Loss = 44.55032\n",
      "140:59 Training NCE Loss = 39.96864\n",
      "0:60 Training NCE Loss = 48.60485\n",
      "10:60 Training NCE Loss = 34.15802\n",
      "20:60 Training NCE Loss = 31.09939\n",
      "30:60 Training NCE Loss = 28.69670\n",
      "40:60 Training NCE Loss = 21.38933\n",
      "50:60 Training NCE Loss = 26.31507\n",
      "60:60 Training NCE Loss = 39.73169\n",
      "70:60 Training NCE Loss = 32.04029\n",
      "80:60 Training NCE Loss = 31.39873\n",
      "90:60 Training NCE Loss = 31.89616\n",
      "100:60 Training NCE Loss = 30.75271\n",
      "110:60 Training NCE Loss = 35.30325\n",
      "120:60 Training NCE Loss = 35.65223\n",
      "130:60 Training NCE Loss = 40.97190\n",
      "140:60 Training NCE Loss = 43.99104\n",
      "0:61 Training NCE Loss = 44.44365\n",
      "10:61 Training NCE Loss = 33.69011\n",
      "20:61 Training NCE Loss = 36.28101\n",
      "30:61 Training NCE Loss = 41.10835\n",
      "40:61 Training NCE Loss = 30.42274\n",
      "50:61 Training NCE Loss = 40.36569\n",
      "60:61 Training NCE Loss = 40.00684\n",
      "70:61 Training NCE Loss = 29.30085\n",
      "80:61 Training NCE Loss = 36.32225\n",
      "90:61 Training NCE Loss = 37.76376\n",
      "100:61 Training NCE Loss = 31.47548\n",
      "110:61 Training NCE Loss = 46.08172\n",
      "120:61 Training NCE Loss = 36.72773\n",
      "130:61 Training NCE Loss = 35.05313\n",
      "140:61 Training NCE Loss = 29.59964\n",
      "0:62 Training NCE Loss = 38.74078\n",
      "10:62 Training NCE Loss = 39.70842\n",
      "20:62 Training NCE Loss = 49.66309\n",
      "30:62 Training NCE Loss = 29.70750\n",
      "40:62 Training NCE Loss = 18.03017\n",
      "50:62 Training NCE Loss = 47.26545\n",
      "60:62 Training NCE Loss = 24.33951\n",
      "70:62 Training NCE Loss = 39.47168\n",
      "80:62 Training NCE Loss = 37.15115\n",
      "90:62 Training NCE Loss = 34.38729\n",
      "100:62 Training NCE Loss = 35.55360\n",
      "110:62 Training NCE Loss = 14.68568\n",
      "120:62 Training NCE Loss = 50.24446\n",
      "130:62 Training NCE Loss = 44.57831\n",
      "140:62 Training NCE Loss = 44.94839\n",
      "0:63 Training NCE Loss = 44.42149\n",
      "10:63 Training NCE Loss = 39.69913\n",
      "20:63 Training NCE Loss = 37.93131\n",
      "30:63 Training NCE Loss = 24.31542\n",
      "40:63 Training NCE Loss = 41.38153\n",
      "50:63 Training NCE Loss = 30.40811\n",
      "60:63 Training NCE Loss = 38.25178\n",
      "70:63 Training NCE Loss = 32.28025\n",
      "80:63 Training NCE Loss = 21.38586\n",
      "90:63 Training NCE Loss = 36.47839\n",
      "100:63 Training NCE Loss = 35.32739\n",
      "110:63 Training NCE Loss = 35.91167\n",
      "120:63 Training NCE Loss = 43.05186\n",
      "130:63 Training NCE Loss = 41.08415\n",
      "140:63 Training NCE Loss = 41.13237\n",
      "0:64 Training NCE Loss = 31.72953\n",
      "10:64 Training NCE Loss = 36.19149\n",
      "20:64 Training NCE Loss = 20.75948\n",
      "30:64 Training NCE Loss = 33.72882\n",
      "40:64 Training NCE Loss = 43.21024\n",
      "50:64 Training NCE Loss = 30.38227\n",
      "60:64 Training NCE Loss = 31.61254\n",
      "70:64 Training NCE Loss = 34.78118\n",
      "80:64 Training NCE Loss = 37.80661\n",
      "90:64 Training NCE Loss = 34.60281\n",
      "100:64 Training NCE Loss = 22.42220\n",
      "110:64 Training NCE Loss = 32.67918\n",
      "120:64 Training NCE Loss = 30.42453\n",
      "130:64 Training NCE Loss = 22.78271\n",
      "140:64 Training NCE Loss = 38.29268\n",
      "0:65 Training NCE Loss = 37.26765\n",
      "10:65 Training NCE Loss = 43.97244\n",
      "20:65 Training NCE Loss = 41.82168\n",
      "30:65 Training NCE Loss = 23.65654\n",
      "40:65 Training NCE Loss = 68.32790\n",
      "50:65 Training NCE Loss = 22.90142\n",
      "60:65 Training NCE Loss = 28.83942\n",
      "70:65 Training NCE Loss = 26.32775\n",
      "80:65 Training NCE Loss = 32.15955\n",
      "90:65 Training NCE Loss = 33.93859\n",
      "100:65 Training NCE Loss = 45.04013\n",
      "110:65 Training NCE Loss = 21.20989\n",
      "120:65 Training NCE Loss = 26.48215\n",
      "130:65 Training NCE Loss = 23.54896\n",
      "140:65 Training NCE Loss = 29.57230\n",
      "0:66 Training NCE Loss = 49.16329\n",
      "10:66 Training NCE Loss = 39.65207\n",
      "20:66 Training NCE Loss = 32.13832\n",
      "30:66 Training NCE Loss = 37.91344\n",
      "40:66 Training NCE Loss = 51.69349\n",
      "50:66 Training NCE Loss = 31.72619\n",
      "60:66 Training NCE Loss = 40.74816\n",
      "70:66 Training NCE Loss = 42.88188\n",
      "80:66 Training NCE Loss = 31.44780\n",
      "90:66 Training NCE Loss = 33.87416\n",
      "100:66 Training NCE Loss = 26.18019\n",
      "110:66 Training NCE Loss = 37.90529\n",
      "120:66 Training NCE Loss = 36.38182\n",
      "130:66 Training NCE Loss = 40.95960\n",
      "140:66 Training NCE Loss = 39.84231\n",
      "0:67 Training NCE Loss = 30.63674\n",
      "10:67 Training NCE Loss = 15.21433\n",
      "20:67 Training NCE Loss = 33.57699\n",
      "30:67 Training NCE Loss = 22.36898\n",
      "40:67 Training NCE Loss = 42.73489\n",
      "50:67 Training NCE Loss = 45.95956\n",
      "60:67 Training NCE Loss = 21.63341\n",
      "70:67 Training NCE Loss = 45.95630\n",
      "80:67 Training NCE Loss = 22.45269\n",
      "90:67 Training NCE Loss = 32.77512\n",
      "100:67 Training NCE Loss = 24.99872\n",
      "110:67 Training NCE Loss = 35.26551\n",
      "120:67 Training NCE Loss = 43.86294\n",
      "130:67 Training NCE Loss = 26.09591\n",
      "140:67 Training NCE Loss = 20.46498\n",
      "0:68 Training NCE Loss = 41.88705\n",
      "10:68 Training NCE Loss = 31.55632\n",
      "20:68 Training NCE Loss = 27.53797\n",
      "30:68 Training NCE Loss = 31.49811\n",
      "40:68 Training NCE Loss = 45.14439\n",
      "50:68 Training NCE Loss = 36.73049\n",
      "60:68 Training NCE Loss = 25.42644\n",
      "70:68 Training NCE Loss = 28.61663\n",
      "80:68 Training NCE Loss = 40.67515\n",
      "90:68 Training NCE Loss = 30.10709\n",
      "100:68 Training NCE Loss = 48.65922\n",
      "110:68 Training NCE Loss = 39.43418\n",
      "120:68 Training NCE Loss = 36.19312\n",
      "130:68 Training NCE Loss = 27.39696\n",
      "140:68 Training NCE Loss = 37.35836\n",
      "0:69 Training NCE Loss = 24.66068\n",
      "10:69 Training NCE Loss = 32.85496\n",
      "20:69 Training NCE Loss = 34.60298\n",
      "30:69 Training NCE Loss = 46.81544\n",
      "40:69 Training NCE Loss = 39.09744\n",
      "50:69 Training NCE Loss = 32.77304\n",
      "60:69 Training NCE Loss = 37.51740\n",
      "70:69 Training NCE Loss = 49.68999\n",
      "80:69 Training NCE Loss = 46.16567\n",
      "90:69 Training NCE Loss = 40.99659\n",
      "100:69 Training NCE Loss = 24.63135\n",
      "110:69 Training NCE Loss = 25.47551\n",
      "120:69 Training NCE Loss = 44.42216\n",
      "130:69 Training NCE Loss = 31.72249\n",
      "140:69 Training NCE Loss = 38.11427\n",
      "0:70 Training NCE Loss = 27.18253\n",
      "10:70 Training NCE Loss = 37.86622\n",
      "20:70 Training NCE Loss = 39.11603\n",
      "30:70 Training NCE Loss = 19.72691\n",
      "40:70 Training NCE Loss = 24.96818\n",
      "50:70 Training NCE Loss = 25.69500\n",
      "60:70 Training NCE Loss = 23.12795\n",
      "70:70 Training NCE Loss = 24.19178\n",
      "80:70 Training NCE Loss = 30.46124\n",
      "90:70 Training NCE Loss = 36.62703\n",
      "100:70 Training NCE Loss = 38.27658\n",
      "110:70 Training NCE Loss = 32.47644\n",
      "120:70 Training NCE Loss = 26.07880\n",
      "130:70 Training NCE Loss = 41.62604\n",
      "140:70 Training NCE Loss = 47.08910\n",
      "0:71 Training NCE Loss = 48.83003\n",
      "10:71 Training NCE Loss = 52.75109\n",
      "20:71 Training NCE Loss = 31.63474\n",
      "30:71 Training NCE Loss = 23.91220\n",
      "40:71 Training NCE Loss = 37.41313\n",
      "50:71 Training NCE Loss = 33.14234\n",
      "60:71 Training NCE Loss = 30.11947\n",
      "70:71 Training NCE Loss = 22.22731\n",
      "80:71 Training NCE Loss = 33.25831\n",
      "90:71 Training NCE Loss = 28.08797\n",
      "100:71 Training NCE Loss = 48.89214\n",
      "110:71 Training NCE Loss = 26.46302\n",
      "120:71 Training NCE Loss = 32.55043\n",
      "130:71 Training NCE Loss = 17.53430\n",
      "140:71 Training NCE Loss = 42.83072\n",
      "0:72 Training NCE Loss = 29.16760\n",
      "10:72 Training NCE Loss = 22.00447\n",
      "20:72 Training NCE Loss = 28.74228\n",
      "30:72 Training NCE Loss = 45.18825\n",
      "40:72 Training NCE Loss = 15.22871\n",
      "50:72 Training NCE Loss = 25.60741\n",
      "60:72 Training NCE Loss = 27.06079\n",
      "70:72 Training NCE Loss = 36.81075\n",
      "80:72 Training NCE Loss = 18.92389\n",
      "90:72 Training NCE Loss = 25.24485\n",
      "100:72 Training NCE Loss = 29.78513\n",
      "110:72 Training NCE Loss = 34.16553\n",
      "120:72 Training NCE Loss = 21.72463\n",
      "130:72 Training NCE Loss = 35.34501\n",
      "140:72 Training NCE Loss = 32.02927\n",
      "0:73 Training NCE Loss = 35.74672\n",
      "10:73 Training NCE Loss = 19.24211\n",
      "20:73 Training NCE Loss = 20.44060\n",
      "30:73 Training NCE Loss = 32.11525\n",
      "40:73 Training NCE Loss = 31.40331\n",
      "50:73 Training NCE Loss = 11.64952\n",
      "60:73 Training NCE Loss = 37.02868\n",
      "70:73 Training NCE Loss = 23.98776\n",
      "80:73 Training NCE Loss = 49.51080\n",
      "90:73 Training NCE Loss = 28.82716\n",
      "100:73 Training NCE Loss = 26.70012\n",
      "110:73 Training NCE Loss = 24.40681\n",
      "120:73 Training NCE Loss = 28.28309\n",
      "130:73 Training NCE Loss = 21.15069\n",
      "140:73 Training NCE Loss = 34.70081\n",
      "0:74 Training NCE Loss = 31.51201\n",
      "10:74 Training NCE Loss = 30.89622\n",
      "20:74 Training NCE Loss = 44.41166\n",
      "30:74 Training NCE Loss = 42.29873\n",
      "40:74 Training NCE Loss = 24.76667\n",
      "50:74 Training NCE Loss = 31.88348\n",
      "60:74 Training NCE Loss = 54.96838\n",
      "70:74 Training NCE Loss = 34.96040\n",
      "80:74 Training NCE Loss = 31.66725\n",
      "90:74 Training NCE Loss = 27.24181\n",
      "100:74 Training NCE Loss = 26.26420\n",
      "110:74 Training NCE Loss = 22.48117\n",
      "120:74 Training NCE Loss = 32.07845\n",
      "130:74 Training NCE Loss = 23.15900\n",
      "140:74 Training NCE Loss = 34.99338\n",
      "0:75 Training NCE Loss = 35.91096\n",
      "10:75 Training NCE Loss = 29.30833\n",
      "20:75 Training NCE Loss = 22.78270\n",
      "30:75 Training NCE Loss = 47.93708\n",
      "40:75 Training NCE Loss = 37.42094\n",
      "50:75 Training NCE Loss = 33.21096\n",
      "60:75 Training NCE Loss = 37.11101\n",
      "70:75 Training NCE Loss = 23.53860\n",
      "80:75 Training NCE Loss = 29.06629\n",
      "90:75 Training NCE Loss = 33.22873\n",
      "100:75 Training NCE Loss = 38.48751\n",
      "110:75 Training NCE Loss = 27.99435\n",
      "120:75 Training NCE Loss = 29.40206\n",
      "130:75 Training NCE Loss = 27.58437\n",
      "140:75 Training NCE Loss = 33.29030\n",
      "0:76 Training NCE Loss = 31.19707\n",
      "10:76 Training NCE Loss = 46.89252\n",
      "20:76 Training NCE Loss = 24.00537\n",
      "30:76 Training NCE Loss = 33.41378\n",
      "40:76 Training NCE Loss = 32.19045\n",
      "50:76 Training NCE Loss = 30.83222\n",
      "60:76 Training NCE Loss = 24.15966\n",
      "70:76 Training NCE Loss = 18.38942\n",
      "80:76 Training NCE Loss = 29.62680\n",
      "90:76 Training NCE Loss = 35.01683\n",
      "100:76 Training NCE Loss = 52.78655\n",
      "110:76 Training NCE Loss = 40.54650\n",
      "120:76 Training NCE Loss = 43.46601\n",
      "130:76 Training NCE Loss = 27.04816\n",
      "140:76 Training NCE Loss = 40.16376\n",
      "0:77 Training NCE Loss = 32.37030\n",
      "10:77 Training NCE Loss = 24.05326\n",
      "20:77 Training NCE Loss = 38.20240\n",
      "30:77 Training NCE Loss = 35.63991\n",
      "40:77 Training NCE Loss = 51.69638\n",
      "50:77 Training NCE Loss = 28.75867\n",
      "60:77 Training NCE Loss = 27.81770\n",
      "70:77 Training NCE Loss = 22.47279\n",
      "80:77 Training NCE Loss = 27.40917\n",
      "90:77 Training NCE Loss = 43.49827\n",
      "100:77 Training NCE Loss = 30.77027\n",
      "110:77 Training NCE Loss = 20.11201\n",
      "120:77 Training NCE Loss = 37.05720\n",
      "130:77 Training NCE Loss = 32.28876\n",
      "140:77 Training NCE Loss = 35.28228\n",
      "0:78 Training NCE Loss = 41.68377\n",
      "10:78 Training NCE Loss = 20.66170\n",
      "20:78 Training NCE Loss = 28.33105\n",
      "30:78 Training NCE Loss = 33.44231\n",
      "40:78 Training NCE Loss = 29.24628\n",
      "50:78 Training NCE Loss = 20.61518\n",
      "60:78 Training NCE Loss = 20.59812\n",
      "70:78 Training NCE Loss = 24.71774\n",
      "80:78 Training NCE Loss = 24.68748\n",
      "90:78 Training NCE Loss = 24.01236\n",
      "100:78 Training NCE Loss = 30.19147\n",
      "110:78 Training NCE Loss = 22.09707\n",
      "120:78 Training NCE Loss = 31.58191\n",
      "130:78 Training NCE Loss = 28.48236\n",
      "140:78 Training NCE Loss = 39.36838\n",
      "0:79 Training NCE Loss = 38.60027\n",
      "10:79 Training NCE Loss = 11.71216\n",
      "20:79 Training NCE Loss = 31.86348\n",
      "30:79 Training NCE Loss = 29.41903\n",
      "40:79 Training NCE Loss = 38.21227\n",
      "50:79 Training NCE Loss = 21.99935\n",
      "60:79 Training NCE Loss = 35.53467\n",
      "70:79 Training NCE Loss = 30.89622\n",
      "80:79 Training NCE Loss = 25.75273\n",
      "90:79 Training NCE Loss = 21.47493\n",
      "100:79 Training NCE Loss = 15.84355\n",
      "110:79 Training NCE Loss = 34.94362\n",
      "120:79 Training NCE Loss = 22.13271\n",
      "130:79 Training NCE Loss = 29.74260\n",
      "140:79 Training NCE Loss = 20.96544\n",
      "0:80 Training NCE Loss = 32.41987\n",
      "10:80 Training NCE Loss = 61.87613\n",
      "20:80 Training NCE Loss = 36.36219\n",
      "30:80 Training NCE Loss = 25.50645\n",
      "40:80 Training NCE Loss = 40.16447\n",
      "50:80 Training NCE Loss = 35.72908\n",
      "60:80 Training NCE Loss = 30.41752\n",
      "70:80 Training NCE Loss = 35.35102\n",
      "80:80 Training NCE Loss = 24.28408\n",
      "90:80 Training NCE Loss = 34.36829\n",
      "100:80 Training NCE Loss = 21.86913\n",
      "110:80 Training NCE Loss = 36.93800\n",
      "120:80 Training NCE Loss = 28.78290\n",
      "130:80 Training NCE Loss = 26.78615\n",
      "140:80 Training NCE Loss = 25.82157\n",
      "0:81 Training NCE Loss = 42.82696\n",
      "10:81 Training NCE Loss = 30.64380\n",
      "20:81 Training NCE Loss = 30.38201\n",
      "30:81 Training NCE Loss = 15.06314\n",
      "40:81 Training NCE Loss = 40.36209\n",
      "50:81 Training NCE Loss = 25.36949\n",
      "60:81 Training NCE Loss = 30.79138\n",
      "70:81 Training NCE Loss = 28.42597\n",
      "80:81 Training NCE Loss = 41.11589\n",
      "90:81 Training NCE Loss = 29.06795\n",
      "100:81 Training NCE Loss = 44.85490\n",
      "110:81 Training NCE Loss = 21.99029\n",
      "120:81 Training NCE Loss = 21.29153\n",
      "130:81 Training NCE Loss = 37.17216\n",
      "140:81 Training NCE Loss = 24.29450\n",
      "0:82 Training NCE Loss = 29.64748\n",
      "10:82 Training NCE Loss = 35.59081\n",
      "20:82 Training NCE Loss = 34.07522\n",
      "30:82 Training NCE Loss = 24.42110\n",
      "40:82 Training NCE Loss = 20.93828\n",
      "50:82 Training NCE Loss = 18.27926\n",
      "60:82 Training NCE Loss = 30.39475\n",
      "70:82 Training NCE Loss = 29.75456\n",
      "80:82 Training NCE Loss = 27.42390\n",
      "90:82 Training NCE Loss = 16.18177\n",
      "100:82 Training NCE Loss = 24.53674\n",
      "110:82 Training NCE Loss = 36.61158\n",
      "120:82 Training NCE Loss = 25.32418\n",
      "130:82 Training NCE Loss = 20.14661\n",
      "140:82 Training NCE Loss = 13.62562\n",
      "0:83 Training NCE Loss = 37.73354\n",
      "10:83 Training NCE Loss = 35.97961\n",
      "20:83 Training NCE Loss = 25.24507\n",
      "30:83 Training NCE Loss = 13.58040\n",
      "40:83 Training NCE Loss = 30.51722\n",
      "50:83 Training NCE Loss = 25.47156\n",
      "60:83 Training NCE Loss = 32.25563\n",
      "70:83 Training NCE Loss = 22.72015\n",
      "80:83 Training NCE Loss = 27.27689\n",
      "90:83 Training NCE Loss = 24.46127\n",
      "100:83 Training NCE Loss = 33.36999\n",
      "110:83 Training NCE Loss = 28.91174\n",
      "120:83 Training NCE Loss = 24.36635\n",
      "130:83 Training NCE Loss = 29.26324\n",
      "140:83 Training NCE Loss = 23.08326\n",
      "0:84 Training NCE Loss = 23.76514\n",
      "10:84 Training NCE Loss = 18.14577\n",
      "20:84 Training NCE Loss = 18.28009\n",
      "30:84 Training NCE Loss = 18.45574\n",
      "40:84 Training NCE Loss = 33.82430\n",
      "50:84 Training NCE Loss = 38.38293\n",
      "60:84 Training NCE Loss = 29.65609\n",
      "70:84 Training NCE Loss = 37.03726\n",
      "80:84 Training NCE Loss = 24.96202\n",
      "90:84 Training NCE Loss = 35.78440\n",
      "100:84 Training NCE Loss = 23.61313\n",
      "110:84 Training NCE Loss = 34.12170\n",
      "120:84 Training NCE Loss = 34.82636\n",
      "130:84 Training NCE Loss = 22.09142\n",
      "140:84 Training NCE Loss = 27.87494\n",
      "0:85 Training NCE Loss = 25.68990\n",
      "10:85 Training NCE Loss = 34.45923\n",
      "20:85 Training NCE Loss = 27.82737\n",
      "30:85 Training NCE Loss = 29.86553\n",
      "40:85 Training NCE Loss = 24.22075\n",
      "50:85 Training NCE Loss = 22.22583\n",
      "60:85 Training NCE Loss = 58.67631\n",
      "70:85 Training NCE Loss = 41.14562\n",
      "80:85 Training NCE Loss = 35.67188\n",
      "90:85 Training NCE Loss = 22.81708\n",
      "100:85 Training NCE Loss = 41.60434\n",
      "110:85 Training NCE Loss = 27.85741\n",
      "120:85 Training NCE Loss = 39.71263\n",
      "130:85 Training NCE Loss = 27.22730\n",
      "140:85 Training NCE Loss = 38.29339\n",
      "0:86 Training NCE Loss = 27.21727\n",
      "10:86 Training NCE Loss = 32.38268\n",
      "20:86 Training NCE Loss = 32.86530\n",
      "30:86 Training NCE Loss = 32.13849\n",
      "40:86 Training NCE Loss = 37.68322\n",
      "50:86 Training NCE Loss = 22.91952\n",
      "60:86 Training NCE Loss = 25.63982\n",
      "70:86 Training NCE Loss = 39.88383\n",
      "80:86 Training NCE Loss = 26.70573\n",
      "90:86 Training NCE Loss = 28.24454\n",
      "100:86 Training NCE Loss = 32.45632\n",
      "110:86 Training NCE Loss = 18.48799\n",
      "120:86 Training NCE Loss = 30.76586\n",
      "130:86 Training NCE Loss = 16.60811\n",
      "140:86 Training NCE Loss = 25.27574\n",
      "0:87 Training NCE Loss = 38.73825\n",
      "10:87 Training NCE Loss = 27.38297\n",
      "20:87 Training NCE Loss = 26.91610\n",
      "30:87 Training NCE Loss = 30.43588\n",
      "40:87 Training NCE Loss = 35.09727\n",
      "50:87 Training NCE Loss = 30.79229\n",
      "60:87 Training NCE Loss = 22.34586\n",
      "70:87 Training NCE Loss = 36.21277\n",
      "80:87 Training NCE Loss = 19.65659\n",
      "90:87 Training NCE Loss = 15.02355\n",
      "100:87 Training NCE Loss = 23.82349\n",
      "110:87 Training NCE Loss = 21.53397\n",
      "120:87 Training NCE Loss = 23.98577\n",
      "130:87 Training NCE Loss = 25.42910\n",
      "140:87 Training NCE Loss = 29.22778\n",
      "0:88 Training NCE Loss = 22.76778\n",
      "10:88 Training NCE Loss = 35.57888\n",
      "20:88 Training NCE Loss = 27.29585\n",
      "30:88 Training NCE Loss = 18.98222\n",
      "40:88 Training NCE Loss = 29.88699\n",
      "50:88 Training NCE Loss = 33.02742\n",
      "60:88 Training NCE Loss = 16.35005\n",
      "70:88 Training NCE Loss = 36.32391\n",
      "80:88 Training NCE Loss = 18.05441\n",
      "90:88 Training NCE Loss = 22.75671\n",
      "100:88 Training NCE Loss = 12.34106\n",
      "110:88 Training NCE Loss = 29.37855\n",
      "120:88 Training NCE Loss = 30.19525\n",
      "130:88 Training NCE Loss = 40.54791\n",
      "140:88 Training NCE Loss = 30.21449\n",
      "0:89 Training NCE Loss = 26.47145\n",
      "10:89 Training NCE Loss = 26.95252\n",
      "20:89 Training NCE Loss = 36.96066\n",
      "30:89 Training NCE Loss = 29.98916\n",
      "40:89 Training NCE Loss = 27.56997\n",
      "50:89 Training NCE Loss = 29.07923\n",
      "60:89 Training NCE Loss = 20.40351\n",
      "70:89 Training NCE Loss = 34.79020\n",
      "80:89 Training NCE Loss = 30.80805\n",
      "90:89 Training NCE Loss = 36.73107\n",
      "100:89 Training NCE Loss = 22.29387\n",
      "110:89 Training NCE Loss = 18.44200\n",
      "120:89 Training NCE Loss = 22.45033\n",
      "130:89 Training NCE Loss = 18.21423\n",
      "140:89 Training NCE Loss = 14.03621\n",
      "0:90 Training NCE Loss = 23.10185\n",
      "10:90 Training NCE Loss = 28.36968\n",
      "20:90 Training NCE Loss = 27.59224\n",
      "30:90 Training NCE Loss = 32.35656\n",
      "40:90 Training NCE Loss = 29.81908\n",
      "50:90 Training NCE Loss = 15.72253\n",
      "60:90 Training NCE Loss = 28.30261\n",
      "70:90 Training NCE Loss = 38.32011\n",
      "80:90 Training NCE Loss = 25.16342\n",
      "90:90 Training NCE Loss = 18.61009\n",
      "100:90 Training NCE Loss = 51.15449\n",
      "110:90 Training NCE Loss = 26.96589\n",
      "120:90 Training NCE Loss = 37.54000\n",
      "130:90 Training NCE Loss = 44.84734\n",
      "140:90 Training NCE Loss = 45.19033\n",
      "0:91 Training NCE Loss = 24.93464\n",
      "10:91 Training NCE Loss = 20.93140\n",
      "20:91 Training NCE Loss = 44.17391\n",
      "30:91 Training NCE Loss = 15.19563\n",
      "40:91 Training NCE Loss = 19.94957\n",
      "50:91 Training NCE Loss = 31.10954\n",
      "60:91 Training NCE Loss = 16.90214\n",
      "70:91 Training NCE Loss = 37.77159\n",
      "80:91 Training NCE Loss = 30.52039\n",
      "90:91 Training NCE Loss = 26.43575\n",
      "100:91 Training NCE Loss = 29.32697\n",
      "110:91 Training NCE Loss = 29.26326\n",
      "120:91 Training NCE Loss = 41.05922\n",
      "130:91 Training NCE Loss = 27.07150\n",
      "140:91 Training NCE Loss = 30.29883\n",
      "0:92 Training NCE Loss = 27.67818\n",
      "10:92 Training NCE Loss = 19.72431\n",
      "20:92 Training NCE Loss = 18.32802\n",
      "30:92 Training NCE Loss = 34.21138\n",
      "40:92 Training NCE Loss = 20.12490\n",
      "50:92 Training NCE Loss = 43.25682\n",
      "60:92 Training NCE Loss = 28.93773\n",
      "70:92 Training NCE Loss = 35.84881\n",
      "80:92 Training NCE Loss = 35.07531\n",
      "90:92 Training NCE Loss = 13.70132\n",
      "100:92 Training NCE Loss = 19.42702\n",
      "110:92 Training NCE Loss = 34.30787\n",
      "120:92 Training NCE Loss = 30.03380\n",
      "130:92 Training NCE Loss = 21.14718\n",
      "140:92 Training NCE Loss = 40.46320\n",
      "0:93 Training NCE Loss = 18.79679\n",
      "10:93 Training NCE Loss = 24.81675\n",
      "20:93 Training NCE Loss = 38.57827\n",
      "30:93 Training NCE Loss = 30.90301\n",
      "40:93 Training NCE Loss = 24.72698\n",
      "50:93 Training NCE Loss = 21.95476\n",
      "60:93 Training NCE Loss = 39.59927\n",
      "70:93 Training NCE Loss = 13.31295\n",
      "80:93 Training NCE Loss = 23.06097\n",
      "90:93 Training NCE Loss = 38.80275\n",
      "100:93 Training NCE Loss = 37.51686\n",
      "110:93 Training NCE Loss = 22.85070\n",
      "120:93 Training NCE Loss = 13.66780\n",
      "130:93 Training NCE Loss = 35.22826\n",
      "140:93 Training NCE Loss = 37.91380\n",
      "0:94 Training NCE Loss = 22.07535\n",
      "10:94 Training NCE Loss = 35.47198\n",
      "20:94 Training NCE Loss = 31.29270\n",
      "30:94 Training NCE Loss = 17.37481\n",
      "40:94 Training NCE Loss = 38.23731\n",
      "50:94 Training NCE Loss = 41.34361\n",
      "60:94 Training NCE Loss = 27.63916\n",
      "70:94 Training NCE Loss = 26.57102\n",
      "80:94 Training NCE Loss = 27.11538\n",
      "90:94 Training NCE Loss = 29.66079\n",
      "100:94 Training NCE Loss = 20.24459\n",
      "110:94 Training NCE Loss = 28.07933\n",
      "120:94 Training NCE Loss = 23.24932\n",
      "130:94 Training NCE Loss = 25.85042\n",
      "140:94 Training NCE Loss = 24.31338\n",
      "0:95 Training NCE Loss = 25.56757\n",
      "10:95 Training NCE Loss = 39.58491\n",
      "20:95 Training NCE Loss = 25.75740\n",
      "30:95 Training NCE Loss = 17.34910\n",
      "40:95 Training NCE Loss = 29.19180\n",
      "50:95 Training NCE Loss = 28.39431\n",
      "60:95 Training NCE Loss = 13.80042\n",
      "70:95 Training NCE Loss = 21.91819\n",
      "80:95 Training NCE Loss = 42.73407\n",
      "90:95 Training NCE Loss = 16.19001\n",
      "100:95 Training NCE Loss = 30.57553\n",
      "110:95 Training NCE Loss = 37.86874\n",
      "120:95 Training NCE Loss = 14.95778\n",
      "130:95 Training NCE Loss = 26.91183\n",
      "140:95 Training NCE Loss = 20.49564\n",
      "0:96 Training NCE Loss = 48.14315\n",
      "10:96 Training NCE Loss = 23.61258\n",
      "20:96 Training NCE Loss = 30.94818\n",
      "30:96 Training NCE Loss = 25.94768\n",
      "40:96 Training NCE Loss = 25.61065\n",
      "50:96 Training NCE Loss = 25.24154\n",
      "60:96 Training NCE Loss = 21.01299\n",
      "70:96 Training NCE Loss = 31.76873\n",
      "80:96 Training NCE Loss = 26.57805\n",
      "90:96 Training NCE Loss = 14.06264\n",
      "100:96 Training NCE Loss = 25.80818\n",
      "110:96 Training NCE Loss = 21.60392\n",
      "120:96 Training NCE Loss = 24.61708\n",
      "130:96 Training NCE Loss = 20.22343\n",
      "140:96 Training NCE Loss = 15.94474\n",
      "0:97 Training NCE Loss = 31.49564\n",
      "10:97 Training NCE Loss = 21.35939\n",
      "20:97 Training NCE Loss = 29.57462\n",
      "30:97 Training NCE Loss = 26.30713\n",
      "40:97 Training NCE Loss = 21.81836\n",
      "50:97 Training NCE Loss = 27.77917\n",
      "60:97 Training NCE Loss = 27.35161\n",
      "70:97 Training NCE Loss = 25.58364\n",
      "80:97 Training NCE Loss = 19.13388\n",
      "90:97 Training NCE Loss = 22.36250\n",
      "100:97 Training NCE Loss = 29.98191\n",
      "110:97 Training NCE Loss = 21.54650\n",
      "120:97 Training NCE Loss = 28.73026\n",
      "130:97 Training NCE Loss = 28.17527\n",
      "140:97 Training NCE Loss = 18.47995\n",
      "0:98 Training NCE Loss = 11.25886\n",
      "10:98 Training NCE Loss = 20.50138\n",
      "20:98 Training NCE Loss = 28.98696\n",
      "30:98 Training NCE Loss = 23.68803\n",
      "40:98 Training NCE Loss = 15.14200\n",
      "50:98 Training NCE Loss = 28.70179\n",
      "60:98 Training NCE Loss = 22.64865\n",
      "70:98 Training NCE Loss = 16.10330\n",
      "80:98 Training NCE Loss = 33.51410\n",
      "90:98 Training NCE Loss = 23.82542\n",
      "100:98 Training NCE Loss = 26.80119\n",
      "110:98 Training NCE Loss = 40.78328\n",
      "120:98 Training NCE Loss = 30.52941\n",
      "130:98 Training NCE Loss = 26.14127\n",
      "140:98 Training NCE Loss = 30.02827\n",
      "0:99 Training NCE Loss = 14.77135\n",
      "10:99 Training NCE Loss = 25.75524\n",
      "20:99 Training NCE Loss = 15.26889\n",
      "30:99 Training NCE Loss = 26.96860\n",
      "40:99 Training NCE Loss = 27.04456\n",
      "50:99 Training NCE Loss = 25.05141\n",
      "60:99 Training NCE Loss = 22.01654\n",
      "70:99 Training NCE Loss = 33.94128\n",
      "80:99 Training NCE Loss = 31.27949\n",
      "90:99 Training NCE Loss = 33.88001\n",
      "100:99 Training NCE Loss = 19.16462\n",
      "110:99 Training NCE Loss = 19.33557\n",
      "120:99 Training NCE Loss = 24.32211\n",
      "130:99 Training NCE Loss = 20.30234\n",
      "140:99 Training NCE Loss = 34.46406\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 100\n",
    "batch_size = config['batch_size']\n",
    "data_size = len(train_x)\n",
    "num_steps = data_size // batch_size\n",
    "for epoch in range(num_epochs):\n",
    "    # shuffle the data order\n",
    "    random.shuffle(train_x)\n",
    "    for step in range(num_steps):\n",
    "        batch = generate_lang_model_batch(step, batch_size, train_x,\n",
    "                                          max_seq_len=max_sequence_len)\n",
    "#         print(batch[0].shape, batch[1].shape, batch[2].shape)\n",
    "        perplexity = drnn.partial_fit(*batch)\n",
    "        if step % 10 == 0:\n",
    "            print(\"%i:%i Training NCE Loss = %1.5f\" % (step, epoch, perplexity))\n",
    "        if epoch*num_steps + step % 20 == 0:\n",
    "            valid_perplexity = drnn.validation_cost(\n",
    "                                    *generate_lang_model_batch(0, len(valid_x), valid_x,\n",
    "                                                               max_seq_len=max_sequence_len))\n",
    "            print(\"%i:%i Validation NCE Loss = %1.5f\" % (step, epoch, valid_perplexity))\n",
    "        if epoch*num_steps + step % 100 == 0:\n",
    "            print(\"Saving model...\")\n",
    "            drnn.checkpoint()\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "drnn.checkpoint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception AssertionError: AssertionError() in <bound method InteractiveSession.__del__ of <tensorflow.python.client.session.InteractiveSession object at 0x1515354d0>> ignored\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'test_data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-259-cd604a9ae83b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpreds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdists\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdrnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0msequences_to_matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_seq_len\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_seq_len\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_probs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msentence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_sentences\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mint2vocab\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpred\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mlast_preds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m \u001b[0mpred\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mpred\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpreds\u001b[0m \u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'test_data' is not defined"
     ]
    }
   ],
   "source": [
    "preds, dists = drnn.predict(*sequences_to_matrix(test_data, max_seq_len=max_seq_len), return_probs=True)\n",
    "for sentence, pred in zip(test_sentences, preds):\n",
    "    print(sentence, [int2vocab[p] for p in pred])\n",
    "    \n",
    "last_preds = [ pred[-1] for pred in preds ]\n",
    "last_dists = [ dist[-1] for dist in dists ]\n",
    "fig, axs = plt.subplots(len(test_data), 1, figsize=(16,16))\n",
    "plt.tight_layout()\n",
    "for i, dist in enumerate(last_dists):\n",
    "    axs[i].set_title(u\" \".join([t.text for t in test_sentences[i]]) + \" __? :: \" + int2vocab[last_preds[i]] )\n",
    "    axs[i].stem(dist)\n",
    "    axs[i].set_xticks(range(len(vocab)))\n",
    "    axs[i].set_xticklabels(vocab)\n",
    "    axs[i].set_xlim([-1, 9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# fig, ax = plt.subplots()\n",
    "# im = ax.imshow(drnn.word_embeddings.eval(), interpolation='nearest')\n",
    "# ax.set_yticklabels(vocab)\n",
    "# # print(vocab)\n",
    "# fig.colorbar(im)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 441,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# plt.imshow(drnn.softmax_w.eval(), interpolation='nearest')\n",
    "# plt.colorbar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# fig, axs = plt.subplots(1,2, figsize=(16,16))\n",
    "# fig.tight_layout()\n",
    "# im = axs[0].imshow(drnn.rnn_i.eval(), interpolation='nearest')\n",
    "# # axs[0].set_ylabel(\"%i\"% i)\n",
    "# im = axs[1].imshow(drnn.rnn_h.eval(), interpolation='nearest')\n",
    "# fig.colorbar(im)\n",
    "# # axs[0].set_ylabel(\"%i\"% i)\n",
    "# # cbar_ax = fig.add_axes([-.1, 0.15, 0.05, 0.7])\n",
    "# # fig.colorbar(im, cax=cbar_ax)\n",
    "# # fig, axs = plt.subplots(8,1, figsize=(16,16))\n",
    "# # fig.tight_layout()\n",
    "# # for i in range(len(outputs)):\n",
    "# #     im = axs[i].imshow(outputs[i], interpolation='nearest')\n",
    "# #     axs[i].set_ylabel(\"%i\"% i)\n",
    "# #     axs[i].set_yticks(range(10))\n",
    "# #     axs[i].set_aspect('auto')\n",
    "# # cbar_ax = fig.add_axes([-.1, 0.15, 0.05, 0.7])\n",
    "# # fig.colorbar(im, cax=cbar_ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[t-SNE] Computing pairwise distances...\n",
      "[t-SNE] Computed conditional probabilities for sample 100 / 100\n",
      "[t-SNE] Mean sigma: 0.068572\n",
      "[t-SNE] Error after 94 iterations with early exaggeration: 17.813980\n",
      "[t-SNE] Error after 210 iterations: 1.456491\n"
     ]
    }
   ],
   "source": [
    "num_points = 100 #len(vocab)\n",
    "a =1000\n",
    "from sklearn.manifold import TSNE\n",
    "\n",
    "embeddings = drnn.word_embeddings.eval()\n",
    "\n",
    "tsne = TSNE(perplexity=5, n_components=2, \n",
    "            init='pca', n_iter=5000,\n",
    "            verbose=1,\n",
    "            learning_rate=100,\n",
    "            random_state=0)\n",
    "nn_2d_embeddings = tsne.fit_transform(embeddings[a:a+num_points])\n",
    "# levy_2d_embeddings = tsne.fit_transform(levy_embeddings[:num_points])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6gAAAIXCAYAAACRqnM8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XtYVWX6//H3AhHII2lO6jRJhzGVzTkEEYEKsAwLTCu1\nQEvrN5UOQ41WTto09R1TnNHKNL8lpPUt03C0ZgTSMLdGCIKamJMa1kwn8wBCmALr9weyE1HzwGZv\n4PO6Lq7Za+2117pXtCbu/TzPfRumaSIiIiIiIiLiaC6ODkBEREREREQElKCKiIiIiIiIk1CCKiIi\nIiIiIk5BCaqIiIiIiIg4BSWoIiIiIiIi4hSUoIqIiIiIiIhTuOgE1TCM1wzD+M4wjO0n7ZthGMZ/\nDMMoOvEz9GKvIyIiIiIiIq1bU4ygLgZOTUBNYI5pmgEnftY0wXVERERERESkFbvoBNU0zQ3AodO8\nZVzsuUVERERERKTtsOca1EcMw9hqGMarhmF0teN1REREREREpBWwV4L6MuAN+APfAGl2uo6IiIiI\niIi0Eu3scVLTNL+vf20Yxv8Cq089xjAM0x7XFhEREREREedgmuZ5Lf20ywiqYRg9T9pMALaf7jjT\nNPXjJD/Tp093eAz60e/C2X70u3CeH/0unOdHvwvn+tHvw3l+9Ltwnh/9Lpzn50Jc9AiqYRj/B0QC\n3Q3D+AqYDkQZhuFPXTXfL4AHLvY6IiIiIiIi0rpddIJqmubdp9n92sWeV0RERERERNoWe1bxlRYk\nKirK0SHICfpdOA/9LpyHfhfOQ78L56Lfh/PQ78J56HfRshkXOjf4oi9sGKajri0iIiIiIiL2ZRgG\npjMUSRIRERERERE5X0pQRURERERExCkoQRURERERERGnoARVREREREREnIISVBEREREREXEKSlBF\nRERERETEKShBFREREREREaegBFVEREREREScghJUERERERERcQpKUEVERERERMQpKEEVERERERER\np6AEVURERERERJyCElQRERERERFxCkpQRURERERExCkoQRURERERERGnoARVREREREREnIISVBER\nEREREXEKSlBFRERERETEKShBFREREREREaegBFVERETsIiEhgeDgYHx8fFi0aBEAHTt2ZNq0afj7\n+xMWFsb333/v4ChFRMSZGKZpOubChmE66toiIiJif4cOHcLLy4uqqipCQkJYv3493bt3Z/Xq1Qwb\nNowpU6bQuXNnnnzySUeHKiIidmAYBqZpGufzGY2gioiIiF3MnTvXNlL6n//8h88//5z27dszbNgw\nAIKCgigtLXVskCIi4lTaOToAERERaX1yc3NZu3YteXl5eHh4EB0dzdGjR3Fzc7Md4+LiQnV1tQOj\nFBERZ6MRVBEREWly5eXleHl54eHhwc6dO8nLy3N0SCIi0gIoQRUREZEmN3ToUKqrq+nfvz9PPPEE\nYWFhQN16pHqGYTTYFhERUZEkERERsbusrCzS0l4BIDV1InFxcQ6OSOTihYeHs3HjRvbt28emTZu4\n++67HR2SiFNRkSQRERFxOllZWSQkJJGTM5ycnOEkJCSRlZXl6LBELtrGjRsB+OKLL3jzzTcdHI1I\n66ARVBEREbGr2NgR5OQMB5JO7MkgJmYV2dkrHBmWyEXr2LEjFRUVhIaG8tlnn+Ht7U1ycjKTJ092\ndGgiTuFCRlBVxVdERERE5ALUr6GeOXMms2fPZvXq1Q6OSKTlU4IqIiIidpWaOhGrNYmqqrptT88p\npKZmODYokSakWYEiTUcJqoiIiNhVXFwcmZkZJxVJylCRJBEROS0lqCIiImJ3cXFxSkql1erUqRNH\njhxxdBgirYKq+IqIiIiIXID6Nah+fn64urri7+/P3LlzHRyVSMumKr4iIiIiIudJvX1FftmFVPFV\ngioiIiIich7qe/tWVc0E6gp/ZWZqbbXIqZSgioiIiIjYmXr7ipybC0lQtQZVRERERNq8hIQEgoOD\n8fHxYdGiRY4OR6TN0giqiIiIiLR5hw4dwsvLi6qqKkJCQli/fj2XXnrpaY/VFF+Rc6MpviIiIiIi\nF2DGjBmsXLkSgH379rFmzRoGDhx4xuNVJEnkl11Igqo+qCIiIiLSpuXm5rJ27Vry8vLw8PAgOjqa\nn3766ayfUW9fEfvQGlQRERERadPKy8vx8vLCw8ODzz77jLy8PEeHJNJmKUEVERERkTZt6NChVFdX\n079/fx5//HHCwsIcHZJIm6UpviIiIiLSJjVcRzpZU3ZFnICKJImIiIhIm6NKvCL2pyq+IiIiIiLn\nIDZ2BDk5w4GkE3syiIlZRXb2CkeGJdKqXEiCqjWoIiIiIiIi4hS0BlVERETEiXXs2JGKigpHh9Hq\npKZOxGpNoqqqbtvTcwqpqRmODUpENMVXRERExJl16tSJI0eOODqMVqlhkaSJWn8q0sQ0xVdExE7K\nysp4+eWXgbqG7vHx8Q6OSERakoSEBIKDg/Hx8WHRokVA3cjotGnT8Pf3JywsjO+//x6AL774grCw\nMHx9fZk2bZojw2714uLiyM5eQXb2CiWnIk5CCaqIyDk4dOgQ8+fPd3QYItJCvfbaaxQUFLB582bm\nzZvHwYMH+fHHHwkLC6O4uJghQ4bYEtfJkyfz0EMPsW3bNnr16uXgyEVEmpem+IqInIO77rqLVatW\n0bdvX9zc3OjQoQPdu3fn008/JSgoiKVLlwJQWFhIamoqFRUVdO/enfT0dC6//HIHRy8ijjZjxgxW\nrlwJwL59+1izZg2RkZEcPXoUgGXLlpGTk8OiRYvo3r073333Ha6urpSXl9O7d29N8RWRFulCpviq\nSJKIyDmYOXMmO3bsoKioiPXr13PbbbdRUlJCz549CQ8PZ+PGjYSEhPDII4+wevVqunXrxttvv82T\nTz7Jq6++6ujwRcSBcnNzWbt2LXl5eXh4eBAdHc3Ro0dxc3OzHePi4kJ1dbUDoxQRcQ5KUEVEzsHJ\nMz5M0yQkJMQ29c7f35/S0lK6dOnCjh07uOmmmwCoqanR9DwRoby8HC8vLzw8PNi5cyd5eXlnPT48\nPJy33nqLMWPG8MYbbzRTlCIizkEJqojIBXB3d7e9dnV1tY18DBgwgE2bNjkqLBFxQkOHDmXBggX0\n79+fvn37EhYWBtRNfatnGIZte+7cuYwePZqZM2dy2223NThORKS1U4IqInIOfqnNg2EY9O3bl/37\n95OXl0doaCjHjx/n888/p3///s0YqYg4m/bt2/PPf/6z0f7y8nLb6xEjRtCxY0diY0cAMH36dFtV\n2WeeeaZ5AhURcQJKUEVEzkG3bt0IDw/HYrHg6el52sJHbm5uLF++nEmTJlFWVkZ1dTUpKSlKUEXk\nF2VlZZGQkERV1UwArNYkMjMz1PpERNocVfEVEWkiavguIhcqNnYEOTnDgaQTezKIiVlFdvYKR4Yl\nInJRVMVXRMRBNPohIiIicvGUoIqINIG0tFdOJKd1ox9VVXX7lKCKyLlITZ2I1ZpEVVXdtqfnFFJT\nMxwblIiIAyhBFREREXGwuLg4MjMzTlomoBkYItI2aQ2qiEgTOHWKr6fnFE3xFRERkTbtQtagKkEV\nEWkiKpIkIiIi8jMlqCIiIiJOoGPHjlRUVPD1118zefJk3nnnHdLT0yksLOSFF15wdHgiIs3iQhJU\nF3sFIyIiItJWGUbd32O9evXinXfeabBPRETOTAmqiIiIiJ2UlpZisVgAOHnm2Pvvv8+gQYM4cOAA\n2dnZDBo0iKCgIEaNGkVlZaWjwhURcTglqCIiIiLNKDMzk5kzZ/Kvf/0L0zR59tlnWbt2LYWFhQQF\nBTFnzhxHhygi4jBqMyMiIiLSTNatW0dBQQE5OTl07NiR9957j5KSEgYNGgTAsWPHbK9FRNoijaCK\nSJtXWVnJsGHD8Pf3x2KxsGzZMtauXUtgYCC+vr7cd999HDt2DIA+ffrwxBNPEBAQQHBwMFu2bCE2\nNpZrrrmGhQsX2s45a9YsQkJC8PPzY8aMGQ66MxFxJoZhcPXVV1NRUcGuXbts+2NiYigqKqKoqIgd\nO3awaNEiB0YpIuJYSlBFpM1bs2YNvXv3pri4mO3btxMXF8e4ceNYtmwZ27Zto7q6mpdffhmo+wPz\nyiuvpKioiCFDhpCcnExmZiZ5eXlMnz4dgOzsbHbv3k1+fj5FRUUUFhayYcMGR96iNIHVq1czc+ZM\nR4chLZhpmlx55ZUsX76ce++9l5KSEgYOHMjGjRvZs2cPUPeF2eeff+7gSEVEHEcJqoi0eb6+vuTk\n5DB16lSsViulpaV4e3tzzTXXAJCUlMRHH31kO3748OEAWCwWwsLC6NChA927d8fd3Z2ysjKys7PJ\nzs4mICCAoKAgdu3axe7dux1yb9J04uPjmTJliqPDkBbi5Iq99a8Nw8AwDPr27csbb7zByJEjqaio\nID09nbvvvhs/Pz8GDRrUYHRVRKSt0RpUEWnzrr32WoqKinj//feZNm0aN9xwQ4P3TdNs8Memu7s7\nAC4uLrRv396238XFherqagAef/xxJk6c2AzRy8UoLS0lPj6e7du3AzB79mwqKyvx8vJi4cKFtGvX\njgEDBvDmm2826GGZnJxMly5dKCgo4Ntvv+X5559nxIgR1NbW8vDDD/Phhx9yxRVX4Obmxvjx4xkx\nYoSD71SaW3l5OVC3LGDbtm1A3ZddSUlJAPj7+7Njxw4AvL29yc/Pd0ygIiJORiOoItLmffPNN3h4\neDBmzBgeffRRPv74Y/bt22ebcrdkyRIiIyMbfe7klhH1DMMgLi6O1157zdYq4r///S/79++3701I\nk6j/ImLmzJkUFxezdetWFixY0OC9et9++y0bN27kvffeY+rUqQC8++677Nu3j507d7JkyRI+/vhj\n9b6U08rKyiI2dgSxsSPIyspydDgiIk5DI6gi0uZt376dxx57zDYi+vLLL3P48GFGjhxJdXU1ISEh\nPPjgg0DjaXunm8YXExPDzp07CQsLA6BTp04sXbqUyy67rBnvSi6Gr68vo0eP5vbbb+f2229v9L5h\nGLb9/fr147vvvgPAarUyatQoAH71q18RHR3dfEFLi5GVlUVCQhJVVXVrmq3WJDIzM4iLi3NwZCIi\njnfRCaphGK8Bw4DvTdO0nNh3KfA2cCVQCowyTfPwxV5LRMQeYmNjiY2NbbR/y5Ytjfbt3bvX9vrk\n6Xqnvjdp0iQmTZrUxJFKU2vXrh21tbW27aqqKgD++c9/sn79elavXs2zzz7L9u3bG42Ynzy9u/49\nwzBOO7IucrK0tFdOJKd1//9RVVW3TwmqiEjTTPFdDAw9Zd9UIMc0zd8Ca09si4i0epq217L86le/\n4vvvv+fgwYP89NNPvPfee9TW1vLll18SFRXFX//6V8rKyqioqDin84WHh7NixQpM0+S7774jNzfX\nvjcgIiLSylz0CKppmhsMw+hzyu7hQP2CrQwgFyWpItLKadpey+Pm5sZTTz1FSEgIvXv3pn///tTU\n1DB27FjKysowTZPJkyfTpUuXM07pPvn1iBEjWLt2Lf379+eKK64gMDCQLl26NPt9iXNLTZ2I1ZrE\niQF7PD2nkJqa4digRESchNEUU5FOJKirT5rie8g0Ta8Trw3gYP32SZ8xNQ1KRFqT2NgR5OQMp37a\nHmQQE7OK7OwVjgxLmlllZSUdOnTgwIEDDBw4kE2bNtGjRw9HhyVOJisri7S0V4C6hFVfZIlIa3Ri\n6ct5VQu0e5Ek0zRNwzBOm4nOmDHD9joqKoqoqCh7hyMiItLkTk42vvtuNy4uLhw7doynnnpKyamc\nVlxcnJJSEWl1cnNzL3p5i71GUD8DokzT/NYwjJ7Ah6ZpXnfKZzSCKiKtyqlTfD09p2iKbxug37uI\niMjpXcgIqr36oK7i5zluScBKO11HRMRpxMXFkZlZN603JmaVkpQ2omFF1rpEtX40VURERM5PU7SZ\n+T/qCiJ1NwzjK+Ap4K/AMsMw7uNEm5mLvY6ISEugaXsiIiIiF+6iR1BN07zbNM1epmm2N03zCtM0\nF5umedA0zZtM0/ytaZqx6oEqIlLn5J6b0jqkpk7E03MKdUXrM05UZJ3o6LBERJpcaWkp/fr1Y+LE\nifj4+BAXF8fRo0fZs2cPN998M8HBwQwZMoRdu3ZRU1PDVVddBcDhw4dxdXXFarUCMGTIEPbs2ePI\nWxEnZq8pviIiLd706dOZO3eubfvJJ59k3rx5PPbYY1gsFnx9fVm2bBlQVxQgPj7eduzDDz9MRkZd\n24g+ffowdepUgoKCWL58efPehNidpnaLSFuye/duHn74YT799FO6du3KihUreOCBB3jhhRcoKChg\n1qxZ/O53v8PV1ZW+fftSUlKC1WolKCiIjz76iJ9++on//Oc/XH311Y6+FXFSdq/iKyLSUo0fP57E\nxEQmT55MbW0tb7/9Ns8//zzvvfce27ZtY//+/Vx//fUMGTKk0WdP7plpGAbdu3ensLCwuW9Bmomm\ndotIW+Ht7Y2vry8AQUFBlJaWsmnTJkaOHGk75tixYwBERETw0Ucf8cUXX/D444+zaNEiIiMjuf76\n6x0Su7QMSlBFRM7gyiuvpFu3bhQXF/Ptt98SEBCA1Wpl9OjRGIZBjx49iIyMZPPmzXTu3Pms57rz\nzjubKWoRERH7cXd3t712dXXlu+++o2vXrhQVFTU6dsiQIcyfP59vvvmGP//5z8yaNYvc3NzTfrEr\nUk9TfEVEzuL+++9n8eLFpKenM378eABObZFlGAbt2rVrsL60qqqqwTEdOnSwf7AiIiLNrHPnzlx1\n1VW2JSymabJ161YAQkJC2LRpE66urri7u+Pn58fChQuVoMpZKUEVETmLhIQE1qxZQ0FBAUOHDiUi\nIoK3336b2tpa9u/fz0cffURISAi/+c1vKCkp4dixYxw+fJh169Y5OnQREZEmV7985eTtpUuX8uqr\nr+Lv74+Pjw+rV68GoH379vzmN78hNDQUqBtRraiowGKxNHvc0nJoiq+IyFm4ublxww034OXlhWEY\nJCQk8PHHH+Pn54dhGMyaNYsePXoAMGrUKHx8fPD29iYwMNDBkYuIiDStPn36sG3bNtt2amqq7fW/\n/vWvRsdnZWXh4XEZ+fk7ycrK4u677+buu+9ullil5TJOnarWbBc2DNNR1xYROVe1tbW26ruqOCgi\nInJusrKySEhIoqpqJgCenlNU5bwNMgwD0zSNXz7yZ5riKyJyBiUlJVx77bXcdNNN552cZmVlERs7\ngtjYEWRlZdkpQhEREeeUlvbKieQ0CahLVNPSXnF0WNICaIqviMgZ9O/f/4IaiZ/6rbHVmqRvjUVE\nRETOgRJUEZEm1vBbY6iqqtunBFVERNqK1NSJWK1J1Be19/ScQmpqhmODkhZBCaqIiIiIiDSpuLg4\nMjMzbNN6U1M1k0jOjYokiYg0MRWGEBEREbmwIklKUEVE7CArK+ukb40nKjkVERGRNkcJqoiIiIiI\niDgFtZkRERERERGRFksJqoiIiIiIiDgFJagiIiIiIiLiFJSgioiIiIiIiFNQgioiIiItRnp6Oo88\n8oijwxARETtRgioiIiIthmGcVzFIERFpYZSgioiISJMpLS3luuuuY+zYsfTv35+RI0dSVVVFYWEh\nUVFRBAcHM3ToUL799lsAiouLCQ0Nxc/Pj8TERA4fPgxAVFQUv//97wkICMBisbB58+ZG19q/fz93\n3HEHISEhhISEsGnTpma9VxERaXpKUEVERKRJ/fvf/+ahhx6ipKSEzp078+KLLzJp0iSWL19OQUEB\n48aN48knnwTg3nvvZdasWWzduhWLxcLTTz8N1I2UVlVVUVRUxPz58xk/fjwAJ/dQnzx5MikpKeTn\n57N8+XLuv//+5r9ZERFpUu0cHYCIiIi0LldccQVhYWEAjB07lmeffZZPP/2UmJgYAGpqaujVqxfl\n5eWUlZUREREBQFJSEiNHjrSd5+677wYgIiLCduzJPvjgA3bu3GnbPnLkCD/++COXXHKJXe9PRETs\nRwmqiIiINKmT14mapknnzp0ZMGBAoym4pyacJ4+Ono6LS8OJX6Zp8sknn9C+ffuLjFhERJyFpviK\niIhIk/ryyy/Jy8sD4M033yQ0NJT9+/fb9h0/fpySkhK6dOmCl5cXVqsVgCVLlhAVFQXUJZ9vv/02\nAFarla5du9KpU6cG14mNjWXevHm27eLiYnvfmoiI2JlGUEVERKRJ9e3bl5deeonx48czYMAAJk2a\nRFxcHJMmTaKsrIzq6mpSUlLo378/GRkZPPjgg/z4449cffXVLF68GKgbhfXw8CAwMJDq6mpee+01\n2/76Edp58+bx0EMP4efnR3V1NZGRkcyfP99h9y0iIhfP+KXpNHa7sGGYjrq2iIiI2EdpaSnx8fFs\n3779os4THR1NWloagYGBTRSZiIg0N8MwME3zvPqDaYqviIiINCl79yrNysoiNnYEsbEjyMrKsuu1\nRESkeWkEVUREWoX6/6bYOzkSx8rKyiIhIYmqqpkAeHpOITMzg7i4OAdHJiIip9IIqoiItGpz5szB\nYrFgsViYO3cu+/bto2/fviQlJWGxWPjPf/7j6BDFztLSXjmRnCYBdYlqWtorjg5LRESaiIokiYhI\ni1BYWEh6ejr5+fnU1tYycOBAIiMj2b17N0uWLCEkJMTRIYqIiMhFUoIqIiItgtVqJTExEU9PTwAS\nExPZsGEDV155pZLTNiQ1dSJWaxJVVXXbnp5TSE3NcGxQIiLSZJSgiohIi3BiHUuj/R06dHBANOIo\ncXFxZGZm2Kb1pqZq/amISGuiIkkiItIiFBUVkZycTF5eHrW1tYSGhrJkyRLuueeei25pIiIiIk1P\nRZJERKTVCggIIDk5mZCQEEJDQ5kwYQJeXl6q2isiF23YsGGUl5dTVlbGyy+/bNufm5tLfHy8AyMT\naXs0gioiIi1OVlbWSVM8J2qKp4g0idLSUuLj422zMnJzc0lLS2P16tUOjkykZdIIqoiItHr1fTBz\ncoaTkzOchIQksrKyHB2WiDixWbNm8cILLwCQkpLCjTfeCMC6desYM2YM3t7eHDhwgKlTp7Jnzx4C\nAgL44x//iGEYVFRUMHLkSPr168fYsWMdeRsibYISVBERaVHUB1NEzteQIUPYsGEDAAUFBVRWVlJd\nXY3VaiUyMhKoG+mZOXMmV199NUVFRTz//POYpklRURFz586lpKSEvXv3snHjRkfeikirpwRVRERE\nRFq1wMBACgsLOXLkCB4eHoSFhVFQUMCGDRuIiIiwHXe65WchISH06tULwzDw9/entLS0GSMXaXvU\nZkZERFoU9cEUkfPl5uaGt7c36enpDBo0CF9fX9atW8eePXvo16/fWT/r7u5ue+3q6kp1dbW9wxVp\n0zSCKiIiLUp9H8yYmFXExKwiM1N9MEXkl0VERDB79mwiIyOJiIhgwYIFBAQENDimU6dOHDlyxEER\nighoBFVEmljHjh2pqKhwdBjSysXFxSkpFZHzEhERwXPPPUdYWBienp54eno2mN4L0K1bN8LDw7FY\nLNxyyy3ccsstjVpZqbWViH2pzYyINCl9+ywiIq2B2lmJXDy1mRGRi1JZWcmwYcPw9/fHYrGwbNky\nvL29OXjwIFBX+TA6OhqAiooKxo0bh6+vL35+fmRmZtrOM23aNPz9/QkLC+P77793yL2ItHZz5szB\nYrFgsViYO3cu+/bto1+/fkycOBEfHx/i4uI4evSoo8MUaZGaq53V66+/jp+fH/7+/tx777289957\nhIaGEhgYSExMjO2/oTNmzGD8+PFER0dz9dVX21rmiLRGSlBFxGbNmjX07t2b4uJitm/fztChQ894\n7DPPPIOXlxfbtm1j69attsS1srKSsLAwiouLGTJkCIsWLWqu8EXajMLCQtLT08nPzycvL49FixZx\n6NAhdu/ezcMPP8ynn35K165dWbFihaNDFWmRmqOd1Y4dO3j22Wf58MMPKS4uZu7cuQwePJi8vDy2\nbNnCnXfeyfPPP287/t///jfZ2dnk5+fz9NNPU1NT06TxiDgLJagiYuPr60tOTg5Tp07FarXSuXPn\nMx67du1aHnroIdt2165dAWjfvj3Dhg0DICgoSOX4RezAarWSmJiIp6cnHTp0IDExkQ0bNuDt7Y2v\nry+g50/E2a1bt45Ro0Zx6aWXAuDl5cVXX31FbGwsvr6+zJ49m5KSEqBumuSwYcNwc3OjW7du9OjR\ng++++86R4YvYjRJUEbG59tprKSoqwmKxMG3aNP785z/Trl07amtrARpNFzzdOnI3NzfbaxcXF5Xj\nP4P6f6YiF+LEmp5G+9tSO4yOHTs6OgRpxVJTJ+LpOQXIADJOtLOa2KTXON1z/MgjjzBp0iS2bdvG\nwoULqarvp0XdF8D1WvvzLW2bElQRsfnmm2/w8PBgzJgxPProoxQVFeHt7U1BQQFAg+mCMTExvPTS\nS7btw4cPN3u8ziwhIYHg4GB8fHxs05w7duzIo48+ir+/Px9//DFLly5l4MCBBAQE8OCDDypplXMW\nERHBypUrqaqqorKykszMzEbVSFs7VVIVe2qOdlY33HAD77zzjq3Ow8GDBykvL6dXr14ApKen245V\nYVFpS5SgiojN9u3bbQnTM888w5/+9CeeeuopJk+ezPXXX0+7du1sfxROmzaNQ4cOYbFY8Pf3Jzc3\nF2j4R6NhGG32j8jXXnuNgoICNm/ezLx58zh48CA//vgjoaGhFBcXc+mll7Js2TI2bdpEUVERLi4u\nvPHGG44OW1qIgIAAkpOTCQkJITQ0lAkTJuDl5dWq2mHMmjXLVggmJSWFG2+8EaibFjlmzBjg9AXZ\nSktLueGGG/Dz8+Omm27iq6++cswNSIsXFxdHdvYKsrNX2KWCb//+/XnyySeJjIzE39+f1NRUZsyY\nwciRIwkODuayyy6zPcNt+b+n0vaozYyINAmV429oxowZrFy5EoB9+/axZs0aBg8ezLFjxzAMgxdf\nfJH/+Z//oUePHgBUVVUxevRonnrqKUeGLS1Ua3z+PvnkE9LS0li2bBkREREcP34cq9XKc889x+WX\nX86DDz4A7aC5AAAgAElEQVTI6tWrGTZsGFOmTKFz5848+eSTxMfHM2rUKO655x4WL17MqlWrGlQZ\nF2lJWuOzLW3LhbSZaWevYESk7agvx19X8RCs1iS7TIdqKXJzc1m7di15eXl4eHgQHR3N0aNH8fDw\naPANeFJSEs8995wDI5XWoLU+f4GBgRQWFnLkyBE8PDwIDg6moKCADRs2MG/evEYF2XJycgDIy8uz\nfTk0duxY/vjHPzrsHkQuRmt9tkV+iab4ishFa45y/C1JeXk5Xl5eeHh4sHPnTvLy8hodc+ONN7J8\n+XL2798P1K09+vLLL5s7VGkFWuvz5+bmhre3N+np6QwaNIjBgwezbt069uzZQ79+/c5akE0ztKQ1\naK3PtsgvUYIqIhesrKyMl19++cTWTiD+vD6fkZHBN9980+RxOdrQoUOprq6mf//+PPHEE4SFhQEN\n1wP269ePv/zlL8TGxuLn50dsbCzffvuto0IWcUoRERHMnj2byMhIIiIiWLBgAQEBAWf9zKBBg3jr\nrbcAeOONNxgyZEhzhCoiIk1EU3xF5IIdOnSI+fPnM3v2bNavv5tjx37Dz+X4M37x8+np6fj4+NCz\nZ0/7B9uM2rdvzz//+c9G+8vLyxtsjxo1ilGjRjVXWNJKpaZOxGpNor4bxbk+fy1BREQEzz33HGFh\nYXh6euLp6WmrVnymgmwvvPAC48aNY9asWfTo0YPFixc7JHaRi9Wan22Rs1GRJBG5YHfddRerVq2i\nb9++/Pjjj+zffxA3N3fc3V0YMmQIS5cuBeCZZ55h9erVVFVVMWjQIBYuXMjy5csZN24cvXv35pJL\nLmHTpk14eHg4+I6ah4peSFPTv1P6ZyCtk/69lpbuQookKUEVkQu2b98+br31VrZv38769eu57bbb\nKCkpoWfPnoSHhzNr1izCw8M5dOgQXl5eANx7772MGjWKW2+9lejoaNLS0ggMDHTwnTSfU4teeHpO\nUdELkYuk50pExDldSIKqNagicsFO/pLJNE1CQkLo1asXhmHg7+9PaWkpUNe3MDQ0FF9fX9atW0dJ\nSclpz9EWqOiFSNPTcyUi0nooQRWRJuPu7m577erqSk1NDUePHuWhhx5ixYoVbNu2jQkTJnD06FHb\ncWo8fmGioqIoLCxstD89PZ1HHnnEARGJiIiIXDwlqCJywTp16sSRI0fOekx9MtqtWzcqKip45513\nGnz+1MJBrV1q6kQ8PacAGfxcUGrieZ2jpqamQVEYkbauKZ4rERFxDkpQReSCdevWjfDwcCwWC3/8\n4x9PmzB17dqVCRMm4OPjw9ChQxk4cKDtveTkZB588EECAwMbjKq2Ztu2bSM5OZGYmFX85jd/oW/f\nnsTFxbFu3TrGjh3L//3f/+Hr64vFYmHq1Km2z3Xs2JFHH30Uf3//Rn1VFy9eTN++fRk4cCCbNm1q\n7lsScbi4uDgyMzOIiVlFTMwqrT8VEWnBVCRJRKQZffLJJ6SlpbFs2TIiIiI4fvw4VquVZ599FsMw\nePXVV9myZQtdu3YlNjaWSZMmcdttt+Hi4sKyZcu44447AGwFpnr27EloaChbtmyhc+fOREdHExgY\nyLx58xx8pyIi0trNmzePBQsWEBQUxJIlSxwdjjghFUkSEaeXlZVFbOwIYmNHkJWV5ehwml1gYCCF\nhYUcOXIEDw8PwsLCKCgowGq10rVrV6Kjo+nWrRuurq6MGTOGjz76CKhb0ztixIgG5zJNk08++YSo\nqCi6deuGm5sbd955Z5srPCUiIo7x8ssv88EHHyg5lSalBFVEmk19K4icnOHk5AwnISGpzSWpbm5u\neHt7k56ezqBBgxg8eDDr1q1j9+7d9OnTp1Fl5Ppp0x4eHqedQn3qPiWnIiJiD3PmzMFisWCxWJg7\ndy7/7//9P/bu3cvQoUP5+9//7ujwpBVRgioizUatIOpEREQwe/ZsIiMjiYiIYMGCBQQGBhISEsL6\n9es5cOAANTU1vPXWW0RGRp7xPIZhMHDgQNavX8/Bgwc5fvx4gyJUIiIiTaGwsJD09HTy8/PJy8tj\n0aJFPPDAA/Tq1Yvc3Fx+//vfOzpEaUWUoIqINLOIiAi+/fZbwsLC6NGjB56enkRERHD55Zfz17/+\nlejoaPz9/QkODiY+Ph44czueyy+/nBkzZhAWFsbgwYMZMGBAs1T37dixo92vISKOERUVxZYtWwDo\n06cPBw8edHBE4mhWq5XExEQ8PT3p0KEDiYmJtiUomrkjTa2dowMQkbYjNXUiVmsSVVV123WtIDIc\nG5QD3HDDDfz000+27V27dtle33XXXdx1112NPnNqO54PP/zQ9jo5OZnk5OSmD/Qs1OJGpPU6+fnW\nsy5gK3TTaN/J/yvSVDSCKiLNRq0gmpY9C07NmjWLF154AYCUlBRuvPFGANatW8eYMWMAmDZtGv7+\n/oSFhfH9998DsH//fu644w5CQkIICQmxtb2ZMWMG48ePJzo6mquvvtp2bhGxn7M9x2PHjiUnJ4dB\ngwYRFBTEqFGjqKysdGS44sQiIiJYuXIlVVVVVFZWkpmZSUREhKPDklZKCaqINKu4uDiys1eQnb1C\nyelFsHfBqSFDhrBhwwYACgoKqKyspLq6GqvVSmRkJJWVlYSFhVFcXMyQIUNYtGgRAJMnTyYlJYX8\n/HyWL1/O/fffbzvnv//9b7Kzs8nPz+fpp5+mpqamyeIVkcbO9Bxv2LABX19f/vKXv/DBBx9QWFhI\nUFAQc+bMcXDE4qwCAgJITk4mJCSE0NBQKisr8ff3P+/zLFy48LQVf0tLS7FYLE0RqrQCmuIrItIC\nNSw4BVVVdfuaKuk/tR1OcHAwBQUFbNiwgXnz5tG+fXuGDRsGQFBQEDk5OQB88MEH7Ny503aeI0eO\nUFlZiWEYDBs2DDc3N7p160aPHj347rvv6NWrV5PEKyKNnek5tlqtDB8+nJKSEsLDwwE4duwYgwYN\ncnDE4sxSUlJISUlpsO+LL744r3M88MADTRmStFJKUEVEpJFT2+H4+vqybt069uzZQ79+/XBzc7Md\n6+LiQnV1NfBzb9b27ds3OufJ+1xdXW2fERH7ONNzvHv3bry9vYmJieHNN990dJjSgmRlZZGW9grr\n1v2D999/H3d3d2bMmMFll13Gp59+SlBQEEuXLgVg6tSprF69mnbt2hEXF8fzzz/PjBkz6NSpE6mp\nqRQWFjJ+/HgMwyA2NtZ2jZqaGqZOncr69ev56aefeOihh5g4caKjblkcQFN8RURaoNTUiXh6TgEy\ngIwTBaea9j/gp2uHExAQcNbPxMbGMm/ePNv21q1bmzQmETk/Z2prFRoaysaNG9mzZw8AlZWVfP75\n5w6OVpzZyUtLamrcSEhIIj8/n+LiYubOnUtJSQl79+5l48aNHDhwgJUrV7Jjxw62bt3KtGnTgLqC\nSvVFlcaNG8dLL71EcXFxg+u8+uqrdO3alfz8fPLz81m0aBGlpaXNfbviQEpQRURaoOYoOHWmdjjQ\nuMpn/fa8efMoKCjAz8+PAQMGsHDhwgbHiUjzOtNz3L17d9LT07n77rvx8/Nj0KBBDSqKi5yq4dKS\ndlRVzWTZsvcICQmhV69eGIaBv78/+/bto2vXrnh4eHDfffeRmZmJp6dng3OVlZVRVlbG4MGDAbjn\nnnts72VnZ/P6668TEBBAaGgoBw8eZPfu3c14p+JomuIrItJCxcXF2bXQ1Nna4Zzc9mbEiBGMGDEC\ngG7duvHWW281Otf06dMbbG/fvr2pwxWR0zjbcxwdHU1+fn6jz5zcxup81xhK2+Pu7m577erqyvHj\nx3F1dSU/P5+1a9eyfPlyXnzxRdauXXvGc5zawubFF18kJibGbjGLc9MIqoiI2I09W+GISNPS8ypn\n03BpSTWenlMYNerW0x5bWVnJ4cOHufnmm5kzZ45tuYdpmpimSZcuXejatSsbN24E4I033rB9Ni4u\njvnz59vqFPz73//mxx9/tOu9iXPRCKqIiNhF/XqluilhYLUmqfetiJPS8yq/pH5pSV2RpGoyMzNw\nd3fHarU2OM4wDI4cOcJtt93G0aNHMU2Tv/3tb7b36pd7LF68uEGRpPr9999/P6WlpQQGBmKaJj16\n9CAzM7N5b1Ycyjh1SL1JT24YpUA5UAMcN00z5KT3THteW0REHCs2dgQ5OcOpb4UDdWtms7NXODIs\nETkNPa8iYg+GYWCa5nkVobD3FF8TiDJNM+Dk5FRERERERKSepphLveZYg6qyjSIibVBztMJpy8rK\nynj55Zdt27m5ucTHxzswImnJ9LyKI53cwiYnZzgJCUlKUtuw5hhB/cAwjALDMCbY+VoiIuJEmqMV\nTlt26NAh5s+f32Tnq6mpabJzScuj51UcqWELm7q10Glprzg6LHEQexdJCjdN8xvDMC4DcgzD+Mw0\nzQ31b86YMcN2YFRUFFFRUXYOR0REmpO9W+G0JXPmzGHx4sVAXRGRvLw89uzZQ0BAADExMQwbNoyK\nigpGjhzJp59+SlBQEEuXLgWgsLCQ1NRUKioqbP0vL7/8cqKioggICMBqtTJ69GhSUlIceYviYHpe\nReRi5ebmkpube1HnsGuRpAYXMozpQIVpmmkntlUkSURE5BwUFhYybtw4PvnkE2praxk4cCBLly7l\nnnvusfWUzc3N5fbbb6ekpISePXsSHh7OrFmzCAkJITIyktWrV9OtWzfefvttsrOzefXVV4mOjmbA\ngAG8+OKLDr5DEWnLTq0i7ek5RaP4rcSFFEmy2wiqYRiXAK6maR4xDKMDEAs8ba/riYiItFZWq5XE\nxEQ8PT0BSExM5KOPPmp0XEhICL169QLA39+f0tJSunTpwo4dO7jpppuAuqm89ccA3Hnnnc1wByIi\nZ3ZyCxuA1FQlp22ZPaf4/grIPNHTqB3whmma2Xa8noiISKt04hvoRvtO5e7ubnvt6upqa3Q/YMAA\nNm3adNpzd+jQoQkjFRG5MJpiLvXsViTJNM0vTNP0P/HjY5rm/9jrWiIiIq1ZREQEK1eupKqqisrK\nSjIzMwkPD+fIkSNn/ZxhGPTt25f9+/eTl5cHwPHjxykpKWmOsEVERM6bvYskiYiIyEUKCAggOTmZ\nkJC6luITJkwgMDCQ8PBwLBYLt9xyC7fccstpR1Xd3NxYvnw5kyZNoqysjOrqalJSUujfv39z34aI\niMgvarYiSY0urCJJIiIizS4rK+ukdV4TNaVORETs5kKKJNm7D6qIiEgDHTt2BKC0tBSLxeLgaNqW\n+kqZOTnDyckZTkJCEllZWY4OS0RExEZTfEVEpFmdbhqqNI+0tFdOtHFIAqCqqm6fRlFFRMRZaARV\nRETOqLKykmHDhuHv74/FYmHZsmX06dOHJ554goCAAIKDg9myZQuxsbFcc801LFy4EICKigpuuukm\ngoKC8PX1ZdWqVQ6+ExEREWkJNIIqIiJntGbNGnr37s37778PQHl5OVOmTOHKK6+kqKiIP/zhDyQn\nJ/Pxxx9TVVWFj48PDzzwAJ6enmRmZtKpUyd++OEHwsLCGD58uIPv5sLU1NTg6urq6DCaRGrqRKzW\nJKqq6rY9PaeQmprh2KBEREROohFUERE5I19fX3Jycpg6dSpWq5XOnTsD2JJNi8VCWFgYHTp0oHv3\n7ri7u1NeXk5tbS2PP/44fn5+xMTE8PXXX/P999878laYM2cOFosFi8XC3LlzG62BnT17Nk8//TQA\nUVFRpKSkcP311zNv3jxHhdzk4uLiyMzMICZmFTExq8jMzND0XhERcSoaQRURkTO69tprKSoq4v33\n32fatGnccMMNALi7uwPg4uJC+/btbce7uLhw/Phx3n33XX744Qe2bNmCq6sr3t7eHD161CH3AFBY\nWEh6ejr5+fnU1tYycOBAIiMjGxxjGIZtfaxhGBw/fpzNmzc7Ily7iouLU1IqIiJOSwmqiIic0Tff\nfIOXlxdjxoyha9eu/O///m+D98/ULqy8vJwePXrg6urKhx9+yL59+5oj3DOyWq0kJibi6ekJQGJi\nIh999FGj406+nzvvvLPZ4hMREZE6SlBFROSMtm/fzmOPPWYbKZ0/fz4jR460vX/yqOPJ22PGjCE+\nPh5fX1+Cg4Pp169fg2NO99qeTvRhs22bpklZWRm1tbW2fVVVVQ3i6dChQ7PEJiIiIj8zzvTtt90v\nbBimo64tIiJtS1FREcnJyeTl5VFbW0toaCiLFy/m5ptvZteuXXTo0IHIyEhuueUWnnrqKaKjo0lL\nSyMwMNDRoYuIiLRYJ74gPq9vo1UkSUREmkVWVhaxsSOIjR1BVlZWs147ICCA5ORkQkJCCA0NZcKE\nCQQHB/PUU08REhJCbGws/fv3b9aY5MzmzZtH//79ueeee+xy/lMLZInzCA8Pd3QIIuJgGkEVERG7\ny8rKIiEhiaqqmUBdexNnqyCblZVFWtorQF07FmeKra3p168fa9eupVevXnY5f2lpKfHx8Wzfvt0u\n5xcRkToaQRUREaeUlvbKieQ0CahLVOuTQWdQn0Dn5AwnJ2c4CQlJzT7KK3UefPBB9u7dy9ChQ5kz\nZw633347fn5+hIWF2RLKGTNmkJaWZvuMj48PX375JaWlpfTr14+JEyfi4+NDXFycrXp0YWEhfn5+\n+Pv7M3/+fIfcm/yyjh07AnUF2oYMGUJAQAAWiwWr1ergyESkuShBFRGRNs/ZE+hzdSHTI//xj3+w\nc+dOO0RzYRYsWECvXr3Izc3liy++ICgoiK1bt/Lcc89x7733Ao2La528vXv3bh5++GE+/fRTunbt\nyooVKwAYN24cL730EsXFxc13M3Le6n+Xb775JkOHDqWoqIht27bh7+/v4MiaRkZGBt98842jwxBx\nakpQRUTE7lJTJ+LpOQXIADLw9JxCaupER4fV6mzcuPG8P5OZmUlJSYkdork4pmmyceNG2zrU6Oho\nDhw4wJEjR876OW9vb3x9fQEICgqitLSUsrIyysrKGDx4MIDd1rZK0wkJCWHx4sU8/fTTbNu2zTay\n2tKlp6fz9ddfOzoMEaemBFVEROwuLi6OzMwMYmJWEROzyunWn7aWBLpjx46sX7+e+Ph4276HH36Y\njIwMAKZOncqAAQPw8/Pjscce4+OPP2b16tU89thjBAQEsHfvXkeFfkanq1fRrl27Bi2C6qfxAri7\nu9teu7q6Ul1dfU7nFOcSERHBhg0b6N27N8nJySxZssTRIQFQWVnJsGHD8Pf3x2KxsGzZMhISEmzv\n5+TkkJiYSG1tLcnJyVgsFnx9ffn73//OihUrKCgoYMyYMQQGBnL06FEKCwuJiooiODiYoUOH8u23\n3wIQFRXFH/7wB66//nr69evH5s2bSUhI4Le//S1/+tOfHHX7Is1CfVBFRKRZxMXFOVVSerL6BPrn\nIknOlUCfq9P1la3vTXvw4EFWrlzJZ599BkB5eTmdO3dm+PDhxMfHk5iY2Nzh/qKIiAjeeOMNpk2b\nRm5uLpdddhmdOnWiT58+vPfeewBs2bKFL7744qzn6dKlC127dmXjxo2Eh4fzxhtvNEf4chG+/PJL\nevfuzf33389PP/1EUVGRU4x8r1mzht69e/P+++8Ddc/R9OnTOXDgAN26dWPx4sXcd999FBcX8/XX\nX9vWTdc/by+++KKthdXx48d55JFHWL16Nd26dePtt9/mySef5NVXX8UwDNzd3dm8eTPz5s3jtttu\no6ioCC8vL66++mr+8Ic/4OXl5ch/FCJ2owRVREQE506gm0KXLl3w8PDgvvvu49Zbb+XWW2+1veds\nI4r1SfWMGTMYP348fn5+dOjQwTYSPGLECF5//XV8fHwYOHAgffv2bfDZU88FsHjxYsaPH49hGMTG\nxp42mRfHq/+9fPjhh8yePRs3Nzc6derE66+/7uDI6vj6+vLoo48ydepUbr31VgYPHsw999zDkiVL\nbL2Wly5dSllZGXv37mXSpEkMGzaM2NhY2znqn7ddu3axY8cObrrpJgBqamoaVK4ePnw4UFcEzMfH\nh1/96lcAXHXVVXz55ZdKUKXVUoIqIiLSipxu+qtpmri6upKfn8/atWtZvnw5L774ImvXrgVOP/Lq\nSCdPNc7MzGz0voeHxxmrLG/bts32OjU1Ffi5hVCPHlfbWgjNnDmziaOWM6murqZdu3P7k7O8vByA\npKQkkpKS7BnWBbn22mspKiri/fffZ9q0adx4443cf//9xMfH4+HhwahRo3BxccHLy4tt27axZs0a\nFixYwLJly3j11VeBn5830zQZMGAAmzZtOu216qeru7i4NJi67uLiQk1NjZ3vVMRxtAZVRESkFbny\nyispKSnh2LFjHD58mLVr12IYBpWVlRw+fJibb76ZOXPmsHXrVgA6depkSwpaI7UQunClpaVcd911\njBs3jr59+zJmzBiys7MJDw/nt7/9LZs3b+bgwYNnbAV0zz33MHjwYJKSkvjhhx+44447CAkJISQk\n5IxJWVZWFrGxI4iNHeGUv6dvvvkGDw8PxowZw6OPPkpRURE9e/akV69e/OUvf2HcuHEAHDhwgOrq\nahITE3nmmWcoKioCGj5vffv2Zf/+/eTl5QFw/PhxpyxYJtLcNIIqIiLSShiGwa9//WtGjRqFj48P\n3t7eBAYGAnDkyBFuu+0224jq3/72NwDuuusuJkyYwAsvvMA777zDVVdd5chbuGAdO3akoqKi0f6U\nlKlUVQ2nroUQVFXVtRWqn86dm5tLWloaq1evbs5wW4w9e/awYsUK+vfvz/XXX8/bb7/Nxo0bWbVq\nFc899xxXXHEFQUFBrFy5kg8//JB7773Xlox99tlnWK1W3N3dGT16NCkpKYSHh/Pll18ydOjQRslY\n/ZcJdS2fwGpNcrqCatu3b+exxx7DxcUFNzc3FixYAMDo0aP54YcfbNPN//vf/zJu3DjbbIa//vWv\nACQnJ/Pggw9yySWXsGnTJpYvX86kSZMoKyujurqalJQU+vfv3+Ca9VPeRdoKJagiIiKtwIEDB7j0\n0ksBmDlz5mmnsH7yySe26a5LlqykR48exMXFsWPHjuYOt8md6Q/4X//6KnbuPP/+sFLH29ubAQMG\nADBgwADbekmLxcIXX3zBvn37ePfdd4GGrYAMw2D48OG2qakffPBBg367R44c4ccff+SSSy6x7WvY\nj7jxlwnOIDY2tsF60npWq5UJEybYtn19fSksLGx0XGJiYoOCZH5+fqxfv77RcR9++CHw8/R0cCMr\nK4u4uDjbeyKtlab4iojIWZWWlmKxWBrtnz59um0NozjW119/zaBBg3jsscfOelxLnu46a9YsXnjh\nBQBSUlK48cYbAVi3bh1jxowBYNq0afj7+xMWFsb3338PQI8enrRr9zB1LYSex8VlInv2FBMUFMTe\nvXsxDIOKigpGjhxJv379GDt2rCNuz2mduvaxffv2QN0XAjU1NRiGccYiWycnn6Zp8sknn1BUVERR\nURFfffVVg/dbsqCgID799NMm/3enJT+vF6q19LuVi6MEVURELsjTTz9tSxLEsXr16sWuXbt46KGH\nznpcwxGquqmU9a11nN2QIUPYsGEDAAUFBVRWVlJdXY3VaiUyMpLKykrCwsIoLi5myJAhLFq0CKgr\najNu3N3ExKyic+e/8qc/Pc6ePXv4+OOP6dmzJ6ZpUlRUxNy5cykpKWHv3r1s3LjRkbfaotS3AgIa\ntAI6NWmNjY1l3rx5tu3i4uJG52qp/YgLCwvJzc3Fzc2tSc/bkp/XC3U+U5lN03S6CuTSNJSgiojI\nL6qpqWHixIn4+PgQFxfH0aNHSU5OZsWKFQBMnTqVAQMG4Ofn94ujeCIXIjAwkMLCQo4cOYKHhwdh\nYWEUFBSwYcMGIiIiaN++PcOGDQPqRrRKS0ttn73uuut4990MOnfuwIwZMwBo3749np6eAISEhNCr\nVy8Mw8Df37/BZ9u6M7XtqX89ffp0CgsL8fPz44knnrC1Ajp13eS8efMoKCjAz8+PAQMG8MorjROt\n+n7EMTGriIlZ5XTrT6X5VFRUcNNNNxEUFISvry+rVq0C6mb09O3bl6SkJCwWC1999RXPPPMM1113\nHREREYwePZq0tDSgbv30zTffTHBwMEOGDGHXrl2OvCU5D1qDKiIiv+jzzz/nrbfe4pVXXuHOO+9k\nxYoVtj9ADxw4wMqVK/nss88AWnVF2JYuNXUiVmsSVVV123UjVBmODeocubm54e3tTXp6OoMGDcLX\n15d169axZ88e+vXr12D0ysXFherq6nM+98nTWF1dXc/rs61Znz59GrTtWbx48WnfO10roOnTpwMn\nr6HE1uLnbFp7P+Lz0ZKf14vl6elJZmYmnTp14ocffiAsLMzWF3b37t0sWbKEkJAQNm/ezLvvvsu2\nbds4duwYgYGBBAcHAzBx4kQWLlzINddcwyeffMLvfvc7LUtpIZSgiojIL/L29sbX1xdoPDrVtWtX\nPDw8uO+++7j11lu59dZbHRSl/JL6EaqfE4aWNUIVERHB7NmzWbx4MT4+PqSkpHD99def9TP10wA7\nduzIr3/9a/7xj39w22238dNPPzXoFytNryVU5XVmLf15vRi1tbU8/vjjbNiwARcXF77++mvbuvIr\nr7ySkJAQADZu3Mjtt99O+/btad++PfHx8QBUVlayadMmRo4caTvnsWPHmv9G5IIoQRURkV906ghT\n1Ymv9E3TxNXVlfz8fNauXcvy5ct58cUX9S21E2vJI1QRERE899xzhIWF4enpiaenJxEREUDjqaf1\n2ye/XvL/2bv3uKiq9fHjnwE1EDHRMi91lDSRy3ARRAFRULl4ShLwDgXaN8oyy8xLd7vrT+mIdsz0\nqJhJkpqmXQSOiQqGIaKgmJkyeiwtb6AgXoD1+wOZQNHUgJmB5/16+XrNntmz51lT7Nlrr7WeZ/ly\nnnrqKd544w2aNWvGF198UWMJDynpUTtMISuvsTPlv9e/Y8WKFZw6dYpdu3Zhbm6Ora0tFy9eBMDK\nypG7xdYAACAASURBVEq/37VJuiofl5eXY2Njoy95JEyLdFCFEEL8LcXFxRQXFzNo0CC8vb3p0qWL\noUMSDVT//v25dOmSfrvqmrKqU8vDw8MJDw8H/pxqCtC1a9dqN09qKuFRmSlYCGE4586do23btpib\nm7N582aOHDlS434+Pj489dRTvPzyy1y5coVvvvmGp556Cmtra2xtbVm9ejVDhw5FKUVubq5+JpAw\nbpIkSYgGpLCwkI8//hioyKZYOdWlvhjiM0X9uNGIkkaj4fz58wwePBgXFxd8fX3517/+Vc/RCXH7\nGmMJj/pmqll5heFU/tZERESwc+dOnJ2dWb58Ofb29tftA+Dh4UFISAjOzs7885//RKvVcvfddwMV\no7CLFy/G1dUVJycnfaIlYfw0hkrPrNFolKSGFqJ26XQ6Bg8eTG5uLqmpqcTGxrJhw4abvqe8vBwz\ns9q5V3WrnykahttNfiKEMQkMDCclJYTK6adQkT02OXmNIcNqcOQ8IepacXExVlZWXLhwgX79+jFm\nzBjWrauYKSH/zxne1WnYt7VuQqb4CtGATJs2jUOHDuHm5kbTpk2xsrJi2LBh7N27F3d3dz777DOg\nIvviyJEjSUlJYcqUKZSXl/PBBx+glOLhhx9mxowZQEXB7KKiIgBWr17NN998w9KlSzl06BARERFc\nuHCBkJAQ4uLiOH/+PIC+4P21nykaFkl+IoS4FY11DaWoPzExMeTl5XHx4kW8vb156aW35bfJxEkH\nVYgGZObMmezbt4/s7Gy2bNnCo48+Sl5eHu3bt8fHx4ft27fj7e2NRqPhnnvuISsri99++w0vLy92\n7dpFq1atCAwM1Ge5vDbpSKXnn3+eiRMnMmLECD755JNqMWRnZ1f7zPT0dHx8fOrtOxD1Q5KfCFPX\nmEt4CNGQrFixQv84MDBcfpsaAFmDKkQDcm0mu5sVnx8xYgQAmZmZ+Pv706ZNG8zNzYmIiGDr1q03\n/ZyMjAx96vZRo0ZVe00K3gshTEFlCY+AgPUEBKyXURYhhDASMoIqRAN2s+LzlWnaa0rRXrU8Q6XK\nsiJ/5zNFwyGjT6IhkOmnQjQs8tvUMMgIqhANiLW1tX4t6K3q2bMnW7Zs4fTp05SVlbFy5Ur69esH\nwH333cdPP/1EeXk5a9eu1XdYe/fuzerVqwFYuXJl7TZCmAQZfRJCCGFs5LepYZARVCEakDZt2uDj\n44NWq8XS0pJ27dr95Xvat2/PjBkz8Pf3RynFI488oi8VM2PGDB555BHuvfdePDw8KC4uBmDOnDlE\nRkby/vvvExQUpE/pDteXI5GC9w2XjD41boWFhSQkJDBu3DjJ4C2EMBry22T6pMyMEOK2lZSUYGlp\nCVSMoCYmJrJ27VoDRyWEqE+GLmslhBDC+N1JmRn5lRCiBp9++ikuLi64uroSFRXFkSNH6N+/Py4u\nLgwcOJD//e9/AERHR/PMM8/g5eVFly5dSE1NJSoqCgcHB8aMGaM/XosWLXjxxRdxcnJi4MCBnDp1\nCoBFixbh6emJq6srQ4cO1a/zjI6O5vnnn8fHx4cuXbqwZk1FXb6oqCi++uor/XEjIiIMUng6KysL\nV1dXXFxcWLBgAbGxsUBF6ZHAwHACA8Ol4L0QDVzVslZTpkzRl5iyt7cnMjJSv1/nzp2ZNm0a7u7u\nrFq1iuTkZLy9vXF3d2f48OH6mRlZWVn4+fnh4eFBcHAwJ06cMFTThBBCGJJSyiD/Kj5aCOOzd+9e\n1a1bN3X69GmllFJnzpxRjzzyiPr000+VUkotWbJEDRkyRCmlVFRUlBo1apRSSqmvvvpKWVtbq717\n96ry8nLl7u6u9uzZo5RSSqPRqISEBKWUUm+//bYaP368UkrpP0MppV577TU1b948/XGHDx+ulFIq\nLy9Pde3aVSml1JYtW/SfXVBQoGxtbVVZWVndfRm3YePGjcrS8j4F8QrilaXlfWrjxo2GDksIUUd0\nOp1ycnJSSimVmpqq7r77bvXrr7+q8vJy5eXlpdLT05VSSnXu3FnNmjVLKaXUyZMnVd++fdWFCxeU\nUkrNmDFDvf322+rKlSvKy8tLnTp1Siml1MqVK9XYsWMN0CohhBC16Wqf77b6ibIGVYhrfP/99wwf\nPpzWrVsDYGNjQ0ZGBuvWrQMgMjKSKVOmABXTFirXazo5OdGuXTscHR0BcHR0RKfT4ezsjJmZmb6s\nS2RkJGFhYQDk5uby2muvUVhYSFFREcHBwfrjDhkyBAB7e3t+//13APr27cszzzzDqVOnWL16NUOH\nDjWa6XJSF1OIxkXdoKwVoC8x5e3tDfxZ1iojI4O8vDz985cvX8bb25sDBw6wb98+Bg4cCEBZWZn+\nWEIIIRoX6aAKcY1ry65Uquk5gGbNmgFgZmZWrcSKmZlZjSVWVJUyLtHR0axfvx6tVsuyZctITU29\n7rjXfvbjjz/O8uXLSUxMJD4+/rbaJoQQdeVWyloBBAQEkJCQUO29ubm5ODo6sn379roPVAghhFEz\njqEXIYxI//79WbVqFWfOnAHgzJkzeHt768uprFixgr59+97WMcvLy1m1ahUACQkJ+Pr6AlBUVES7\ndu24cuUKn3322S1lvI2OjmbOnDloNBq6d+9+W3HUpUmTYrC0nAosA5ZdrT0WY+iwhKhVly5dom/f\nvpSXlxs6FIO7k7JWvXr1Ij09nUOHDgFQXFzMwYMH6d69OydPniQjIwOAK1eukJeXV+sxCyGEMH4y\ngirENRwcHHj11Vfp168f5ubm9OjRg3nz5jFmzBhmzZpF27ZtWbp0qX7/qp3KG3Uwrays+PHHH3n3\n3Xe57777SExMBOCdd96hV69e3HvvvfTq1YuioqK/PG7btm1xcHAgNDS01tpcGyprj8XGLgRg0iSp\nPSYanrvuugtfX1/WrVunn6rfWN1JWat7772X+Ph4Ro0axaVLlwB47733eOihh1i9ejUTJkygsLCQ\n0tJSJk6ciIODQ103QwghhJGRMjNC1IM7GWm4VlJSErGxCykrK2X//kwOHDiAtbV1LUUohLhVO3bs\nYPbs2fpZEeLvqTy3QcVMDLmxJYQQDYeUmRHCSN3K1N2bSUpKIjQ0ipSULnz/fTqnThXJWi0hDMTV\n1VX+/mrJn+e2EFJSQggNjZISVUKYkLlz5+Lg4MBjjz1m6FBEAyIjqEKYgMDAcFJSQqjMkAvLCAhY\nT3LyGkOGJUSj1b59e/Lz87GwsDB0KCZNzm1CmDZ7e3s2bdp0S1m3S0tLadJEVhc2Nncygir/lwgh\nhBC3qWo2biGEaIyefvppDh8+THBwMNHR0WzdupX8/HyaN2/OwoUL0Wq1TJ8+nUOHDpGfn0+nTp1Y\nsWKFocMWJkCm+AphAuo6Q+6RI0f4/PPPa+14QjRkly5dwtzcvFpZFXFn7vTcFhoaioeHB05OTixa\ntIjy8nKio6PRarU4OzszZ86cOo9diMZuwYIFdOjQgdTUVPLz83F3d2fPnj28//77PP744/r9fvrp\nJzZt2iSdU3HLZARVCBNQ1xly8/PzSUhIYNSoUbV2TCEaquzsbLy8vAwdRoNwp+e2JUuWYGNjQ0lJ\nCZ6enri7u/Pbb7+Rm5sLQGFhYZ3GLYT4k1KK9PR0vvzySwD8/f05ffo058+fR6PREBISIjf0xG2R\nEVRhcDqdDq1Wa+gwjF5QUBDJyWtITl6DnZ0d3bt3Z8yYMdjZ2REREUFycjI+Pj5069aNzMxMzpw5\nw5AhQ3BxccHLy0t/4bZlyxbc3Nxwc3PD3d2doqIipk2bxrZt23BzcyMuLs7ALRXCOCQlJREYGE5g\nYHi1xD3r168nIiLCgJE1LFXPbbd64y0uLg5XV1e8vLw4duwYly9f5vDhw0yYMIGkpCRatmxZx1EL\nIa51o9wyzZs3r+dIbm7AgAEcP37c0GGIm5ARVCFM1KFDh1izZg0ODg707NmTxMRE0tPTWb9+Pe+/\n/z4PPPAA7u7urFu3js2bN/P444+TnZ1NbGws8+fPx8vLiwsXLnDXXXcxc+ZMZs+ezYYNGwzdLCGM\nQmV22ZKSmQCkpUWxdu0y/Pz8SEtL47333jNwhI1XamoqmzZtIiMjAwsLC/z9/bl8+TI5OTls3LiR\nBQsW8MUXX7B48WJDhypEo+Hr68uKFSt47bXXSE1N5d5778Xa2vqGnVZDKS8v59ChQ7Ru3drQoYib\nkBFUYRRKS0uJjIzEwcGBYcOGUVJSQlZWFn5+fnh4eBAcHMyJEyeAio7ZoEGD8PDwoG/fvhw4cACA\n6Ohonn/+eXx8fOjSpQtr1jTsLJC2trY4Ojqi0WhwdHRk4MCBAGi1WvLz80lLS9Onfa863cbHx4eJ\nEycyb948zp49i7m5udH9gAhhaLGxC692TqOAio5qbOxC7rrrLrZu3SoJkgzo3Llz2NjYYGFhwU8/\n/URGRgYnT56ktLSUsLAw3nnnHXbt2mXoMIVoFDQaDRqNhunTp5OVlYWLiwuvvPIKy5Ytq/Z6fbvR\nDJj9+/czdOhQmXJs5GQEVRiFAwcOsGTJEry8vHjiiSf46KOPWLduHV999RX33HMPiYmJvPrqqyxe\nvJiYmBg++eQTunbtyo4dO3jmmWfYtGkTACdOnCA9PZ39+/cTEhJCeHi4gVtWd6qeXM3MzGjWrBlQ\n8WNQVlZWY8dTo9EwdepUHnnkEb755ht8fHyk5qAQwqQEBwezYMECHBwcsLOzw8vLi19//RV/f3/K\ny8sBmDFjhoGjFKJxOHz4sP7x2rVrgYrO4YsvvglUJEKrzZwZt+JGM2CCgoJwdHRk9uzZ9RqPuH3S\nQRVG4YEHHtAnHYmMjOS9995j7969BAQEAFBWVkaHDh0oLi5m+/btDBs2TP/ey5cvAxWdryFDhgAV\ndbl+//33em6Fcalpuk2LFi04dOgQjo6OODo6kpmZyYEDB7j//vs5f/68oUMWwmhMmhRDWloUJSUV\n2xXZZZcZNqh64OPjQ3p6uqHDuKlmzZrx7bffXvf8hAkTDBCNEKKqm3UO60v1GTBQUlLxXH13lMWd\nkw6qMApVp38opWjZsiWOjo5s37692n6VU7uys7NrPE7lKGLlcRqya6fMVN3WaDS8+eabjB07FhcX\nF6ysrPTTbeLi4ti8eTNmZmY4OTkxaNAgNBoN5ubmuLq6MmbMGJ5//vl6bYsQxqauM2cbK2PvnF4r\nKSmpyn+j+h+pEUJUJ51DURukgyqMwtGjR8nIyKB3794kJCTQu3dvFi1apH/uypUrHDx4EAcHB2xt\nbVm9ejVDhw5FKUVubi7Ozs6GbkK96ty5Mzk5OfrtpUuX1vha5XSbqubOnVvjMSunSQshKgQFBTW6\ni6oWLVpQVFREamoq06dP595772Xv3r24u7vz2WefGTq8aoxhpEYIYXwa6wyYhkSSJAmD02g02NnZ\n8e9//xsHBwcKCwuZMGECq1evZurUqbi6uuLm5sYPP/wAwIoVK1i8eDGurq44OTmxfv36aseq6bGo\n7kbJA4QQjVvV8+bu3buJi4sjLy+Pw4cPG93o6o0SWQkhDGfSpBgsLacCy4BlVzuHMfUaQ+UMmICA\n9QQErJcbVyZIRlCFwXXq1In9+/df97yLiwtbtmy57vnOnTvz3XffXfd81VFEqJgOLK4now5CiFvh\n6elJhw4dAHB1dUWn0+Hj42PgqIQQxsxYlkc0xhkwDYl0UIXJkzVIt0fWhwghbkXVTOHm5uaUlpYa\nMJrryTQ+IYyTdA7F3yVTfIVJqxwNTEkJISUlhNDQKJmyKoQQjYBM4xPi7yksLOTjjz8GIDU1lcGD\nBxs4IiEqyAiqMGkyGnj7ZNRBCHEjN1vHb4zr+mWkRog7d/bsWebPn8+4ceMMHYoQ1WgMVYpDo9Go\nhl4GRNS9wMBwUlJCqOygQsXd9OTkNYYMy+jJtGghhBCicRs5ciTr16/Hzs6Opk2bYmVlxT333HNd\n5u6srCwmTZpEUVER99xzD/Hx8bRr1w4/Pz969+7N5s2bKSgoYPHixfTp08fArRLGRqPRoJS6rTuc\n0kEVJu3ahD+WllNlmpcQQtwhuXklRONx5MgRHnnkEXJzc9myZQuPPvooeXl5tG/fHh8fH2bNmoWn\npyf9+vVjw4YNtGnThsTERJKTk1m8eDH+/v54eHgwa9YsvvvuOz788ENSUlIM3SxhZO6kgypTfIVJ\nM5ZscUIIYeokw7cQjUvVgSKlVI2Zu++++2727dvHwIEDASgrK9PvAxAWFgZAjx490Ol09Re8aNCk\ngypMnqxBEkKIv+9O1/TrdDoGDx5Mbm5uPUQphKgrN8rc7ejoyPbt22/6HmPM9C1Ml2TxFUIIIYQQ\nopGxtrbm/PnzN3xdo9FgZ2fHyZMnycjIAODKlSvk5eXVV4iikZIOqhBCCCGYNCkGS8upwDJg2dUM\n3zH6129WkqKsrIyYmBicnJwICgri4sWLLFq0CE9PT1xdXRk6dCglJSUUFhbSuXNn/fuKi4v5xz/+\nQVlZGYcOHWLQoEF4eHjQt29fDhw4UE8tF6JxatOmDT4+Pmi1WqZMmVJjpu6mTZuyevVqpk6diqur\nK25ubvzwww81Hs8YM30L0yRJkoRR8vPzIzY2Fnd3d0OHIoQQjcbNkiRVncqbmppKbGwsGzZsQKfT\n8dBDD5GVlYWzszMjRowgJCSEQYMG0bp1awBef/117rvvPsaPH8+QIUN44YUX8PPzIzExkU2bNrFw\n4UIGDBjAJ598QteuXdmxYwevvPIKmzZtMsj3IIQQonZIFl9hMO+//z6vvPJKrR3P39+f2NhYevTo\nUWvHFMaj8m9f7rYKYTpuVJIiOzubP/74g3PnzgEwYcIENmzYQLNmzfjjjz9o164dRUVFXLx4kZMn\nT/L555+zfv16fv75Z/7xj38wfvx4evXqxb333kv37t31n3f58mX27dtnqOb+pdtde7tlyxaaNWuG\nl5dXHUfWeCxbtozAwEDat29v6FAaFcn2LW7HnXRQZYqvqBUffPABOp2O7t27M2bMGOzs7IiIiCA5\nORkfHx+6detGZmYmmZmZeHt706NHD3x8fPj5558BKCkpYeTIkTg4OBAWFkZJSYn+2MnJyXh7e+Pu\n7s7w4cMpLi42VDMbtQ8//BCtVotWqyUuLo6XX36Z+fPn61+fPn06sbGxAPrU9C4uLkyfPh2ouJiz\ns7MjKioKrVbLsWPHDNEMIcQdmjlzJl26dCE7O5tZs2aRnZ1NXFwcKSkplJaWkp6ezpUrV1i/fj3D\nhw/nypUrvPLKK3h7e/Puu+/SpEkT9uzZw+DBg/nuu+8YOXIku3bton///pSXl2NjY0N2drb+nzF3\nTu/E5s2bb5hoRtyZ+Ph4fvvtN0OH0ahUZvtOSQkhJSWE0NAokpKSDB2WaGiUUgb5V/HRwhQNGTJE\nubu7K0dHR7Vw4UI1bdo0ZW5uruzt7ZVGo1F79+5V5eXlyt3dXY0dO1YppdRXX32lhgwZos6fP69K\nS0uVUkqlpKSo8PBwpZRSsbGx6oknnlBKKZWTk6OaNGmisrKy1MmTJ1Xfvn3VhQsXlFJKzZgxQ739\n9tsGaHXjtnPnTqXVatWFCxdUUVGRcnR0VNnZ2apfv376fRwcHNSxY8dUUlKSiomJUUopVVZWph55\n5BG1detWlZ+fr8zMzNSOHTsM1AohxN+Rn5+vnJyclFJKbd68WQUEBOifb926tfrss89Ubm6usrCw\nUO3atVNNmjRR9vb2KjAwUA0cOFD17dtXPf/886qsrExZWVmpYcOGqWeffVZ/fG9vb7Vq1SqllFLl\n5eVqz5499d/I25Cfn6+6d++uIiIilL29vRo6dKgqLi5WnTp1UqdPn1ZKKZWZman8/PyUTqdT7dq1\nUx07dlSurq5q27ZtBo7eOFV+p08++aRydHRUgYGBqqSkRGVnZ6tevXopZ2dnFRoaqs6ePatWrVql\nWrRooezs7JSbm5sqKSkxdPiNQkBAmIJ4Berqv3gVEBBm6LCEEbva57utfqKMoIrbtmTJEnbu3Elm\nZiZz585l8uTJWFpa8u2339K1a1ccHR3RaDQ4Ojrq62Y5OTmh0+koKChg6NChaLVaXnzxRX0muG3b\nthEZGQmAVqvF2dkZgIyMDPLy8vD29sbNzY1PP/2Uo0ePGqbhjVhaWhphYWFYWlpiZWVFWFgYW7du\n5Y8//uD48ePs2bMHGxsbOnbsSHJyMsnJybi5ueHu7s6BAwf45ZdfAOjUqROenp4Gbk3Dd+1o95Ej\nR7C3t78uiQ0giWnEHatakkKj0ehLTNx33308/fTTzJs3j4sXL1JQUIC9vT2dO3fmu+++4+uvv8bF\nxYU1a9YwYsQI/TFWrFjB4sWLcXV1xcnJifXr19d7m27XgQMHePbZZ8nLy6Nly5bMnz+/xqULnTp1\n4umnn+bFF18kOzubPn36GCBa0/DLL78wfvx49u7dS6tWrVizZg1RUVHMmjWLPXv2oNVqeeuttxg6\ndCgeHh4kJCSwa9cuLCwsDB26EKKW1FkdVI1GEwzMAcyB/yilZtbVZ4n6FRcXx7p16wA4duwYBw8e\n1L9W9YLFzMyMZs2a6R+Xlpby+uuvM2DAANauXYtOp8Pf31+/v7pmTXLldkBAAAkJCXXWHvHXrq4f\n0G8rpdBoNAwbNozVq1dz4sQJRo4cqX/95ZdfJiYmptoxdDodVlZW9RZzY5WVlUV8fDw//vgj5eXl\n9OrVi379+vHLL7+QmJjIwoULGTFiBGvWrCEiIoKYmJhqiWmeeeYZSUwjanSjkhSdO3dm1KhR+pIU\nTZs2JSgoiN69e/PEE09w8OBBHBwcgIr1qePGjWPJkiXXrVur7MCakgceeEC/pjQyMpK4uLib7n/t\n75y4nq2trf4mtbu7O4cOHaKgoABfX18AoqKiGDZsmH5/+U7r16RJMaSlRVG5Eqsi2/cywwYlGpw6\nGUHVaDTmwEdAMOAAjNJoNPZ18VmifqWmprJp0yYyMjLYvXs3rq6u+pGYv6KU4ty5c3To0AGoWDtS\nqW/fvvpO6N69e8nJyUGj0dC7d2/S09M5dOgQUFGSoGqHWNQPX19f1q1bR0lJCcXFxaxbt46+ffsy\nYsQIPv/8c1avXq2/YAgKCmLJkiX6tcK//vorJ0+eNGT4jUpNo93btm277qJPp9NRXFzM9u3bGTZs\nGG5ubjz99NOcOHHCwC0Qxqo2SlKMHj0aMzMzAgMD9c8lJSURGBhOYGC4ya1lq/odKKUwMzOjSZMm\nlJeXA9zy76P4U9Ub3ebm5hQUFFR7/doOqSTbq19BQUGsXbuMgID1BASsZ+3aZZIkSdS6uhpB9QR+\nUUrpADQazUrgUWB/HX2eqCfnzp3DxsYGCwsL9u/fry/c3LRpU0pLS6/7oai6bWZmxuTJk4mKiuLd\nd9/l4Ycf1r8+btw4xowZg4ODA/b29nh4eABwzz33EB8fz6hRo7h06RIA7733Hg899FB9NFdc5ebm\nRnR0tH567pNPPomLiwsARUVF3H///dx3331AxYj3/v379aMK1tbWfPbZZ2g0GrmQqAfXjnZXuvai\n7+LFi9US04jr2drakp+fb+gwjMqKFStqfH7evHn6xy4uLmzZsqXa65VZP3W6n+nXr5/+XFCZcKWk\npGKSVVpalEld8B49epSMjAx69+5NQkICffr04fz58+zcuZPg4GDWrFmj39fa2lqf6VjcurvvvpvW\nrVuTlpZGnz59WL58OX5+foB8p4YSFBRkMn+jwkTd7qLVW/kHDAUWVdmOBOZds08tL8EV9eHSpUtq\n0KBByt7eXg0ZMkT5+/ur1NRUNXXqVGVvb68iIyMNHaIwMhs3blQBAWEqICBMbdy40dDhNHi7du1S\nzs7O+oRWTk5OKjs7W5/cRimlZs+eraZPn66UMr3ENPWpc+fOhg6hQdi4caOytLxPQQ8FDygLi3v1\n5wJTTrii0+lU9+7dVWRkpD5JUklJidq2bZvq1q2b8vDwUC+99JLy9/dXSin1888/K2dnZ+Xq6qrS\n0tIMHL1xys/PV1qtVr89e/Zs9dZbb6ndu3er3r1765MkFRQUKKWUWrNmjUGTJFVNHCaEqBl3kCSp\nrkZQZUFAA9WsWTO+/fZb/d3wJk1suHjxIjNmzGDGjBm1+llSZ8v0mfroiCmqabTbxsbmhrMbVqxY\nwbhx43j33Xe5cuUKo0aN0k8Fbuzatm1r6BAahNjYhVfPAVEAXLy4jNjYhSZ/HujUqRP7918/MaxP\nnz76ZGNJSUns2XOYwMBwJk2KYc+ePfUdpknp3LkzOTk5+u1JkybpH//www/664Jhw8YyaVIMYWFh\nhIWFGSJUIUQdqqsO6q/AA1W2HwCuK3pYWR8RwM/PTz9lQxi3+uh0SMemYbj2wrSkhAZxYWrsJk6c\nyMSJE6s9d6OLPlNMTFNfduzYYegQGryGnHBFfsdql7F+n6WlpURGRrJr1y4cHR0ZO3YsCxcuZO3a\ntQCkpKTw8ccf8+WXXxo0TiHqS2pqKqmpqX/vILc75Hor/6jo+B4COgPNgN2A/TX71Olwsqg79TEl\ny5SnfYk/yX9H4yVTr0V9+XOKb7yCeGVpeV+1/+ca6v+Lcv6rXcb4febn5yuNRqO2b9+ulFJq7Nix\natasWap79+7q5MmTSimlRo0apb7++mtDhimEQWEsdVCVUqXAeCAJyAMSlVKSIEmIRmbSpBgsLacC\ny4BlV0dHYv7qbeIGdDodWq32bx+nciQiJSWElJQQQkOjTC57qjAdf5X1MygoiOTkNSQnrzH4aJgQ\nt+vaUkPp6ek8/vjjfPbZZxQUFPDll18yaNAgjhw5wueff27gaIUwDXVWB1Up9R0g88YaoPqYktWQ\np301JpUXpn+uJTb8dCwhU69F/WuMWT/ld6x2Gev3eW2pIY1GQ3R0NIMHD8bCwoIXXngBMzMz8vPz\nSUhIYNSoUQaMVgjTUCcjqKJhq48aWFJnq+FoaKMjn376KS4uLri6uvL444/z9ddf07t3b3r0FWW1\n3gAAIABJREFU6EFAQAB//PEHULHGPjY2Vv8+Jycnjh49SnFxMQ8//DCurq5otVq++OILALKysvDz\n88PDw4Pg4GB9PdKsrCz9582fP7/+GyyEuCPyO1a7jPX7rCw1BJCQkICvry/t27enQ4cOvPvuu/oS\nTNOmTWPbtm24ubkRFxfHvn378PT0xM3NDRcXF3755RdDNkMIo6JRNdTLq5cP1miUoT5bCCHuxL59\n+wgLC+OHH36gdevWnD17Fo1GQ6tWrQD4z3/+w08//cTs2bN56623aNGihT4hkVar5euvv2bnzp0k\nJSWxcGHFqPK5c+ewtLSkX79+bNiwgTZt2pCYmEhycjKLFy/G2dmZ+fPn06dPH6ZMmcJ3331Hbm7u\n32rHtclGLC2nGs3FniFIxnAhxJ04cuQIwcHBeHh4kJWVhaOjI8uXL8fCwoKVK1cyd+5ccnNzOX/+\nPFu2bGH27Nls2LABgAkTJtC7d29Gjx5NaWkppaWlWFhYGLhFQtS+q/XZNX+955/qbIqvEEI0NN9/\n/z3Dhw+ndevWANjY2JCbm8vw4cM5ceIEly9f5sEHH7zh+zUaDc7Ozrz00ktMmzaNRx55hD59+rB3\n71727dvHwIEDASgrK6NDhw4UFhZSWFhInz59AHjsscdqJeOuTL3+k7FmBhVCGL8blRoCSEtL48kn\nn2TChAkAXDso4+XlxXvvvcexY8cICwuja9eudR6vEKZCpvgKIcQtunoXsNpzzz33HBMmTCAnJ4dP\nPvmEkqsLpJo0aUJ5ebl+v4sXLwLw0EMPkZ2djVar5bXXXuOdd94BwNHRkezsbLKzs8nJyWHjxo3X\nfVZtzjppaFOv71T19bgVHdXKjrsQQtzMtYnrnnzySbp0caBJk6asXLmSuLg4Lly4QGZm5nXvHTVq\nFBs2bMDS0pJ//vOfbN68uT5DF8KoSQdVCCFuUf/+/Vm1ahVnzpwB4MyZM5w7d44OHToAEB8fr9+3\nc+fO7Nq1C4Bdu3aRn58PwPHjx7GwsCAiIoKXXnqJ7Oxs7OzsOHnypH4d05UrV8jLy6NVq1a0atWK\n9PR0AFasWFFfTRVCCHEbkpKSiI9fyeHDdpSVdaWg4AIzZ87krrvuYuzYsVhbW3P+/Hn9/vn5+dja\n2vLcc8/x6KOP/u2lG0I0JDLFVwghbpGDgwOvvvoq/fr1w9zcHDc3N6ZPn86wYcOwsbGhf//+HDly\nBIDw8HA+/fRTnJyc6NWrF3Z2dgDk5uYyefJkzMzMaNq0KQsWLKBp06asXr2aCRMmUFhYSGlpKRMn\nTsTBwYGlS5cyduxYNBoNgYGB1TJGir/PWDODCiFMS2zsQkpLHwbsgQLKyh4lNnYhTZo04dy5c9ja\n2mJubo6rqyvR0dFcunSJ5cuX07RpU9q3b8+rr75q6CYIYTQkSZIQQhghSdxTf+S7FkLciWPHjhEU\nFMS+ffsIDAwnJQXAGUgFehEQcJDk5DV06tSJvXv3sn37djnXiEbnTpIkSQdVCCGMjGTZFUII43fl\nyhU6dOjAgQMHSE9P59FHw1HqYeAnzM2P8M03a7GysuLZZ5/l//2//yfnddEoSQdVCFErfHx89Ose\nb2TOnDk89dRTWFpa1lNUjUfFnfgQKhL3AFTU/ktOXmPIsIQQQlxj3rx5xMXF0bFjRywsLPjll/9x\n9uxJ/Px80el0lJaWsmTJEl555QM5r4tG6U46qJIkSQhxnb/qnAL67IRCCCFEY/Xcc8/xyy+/sGXL\nFpKSkpg//18A/PZbAR988AE5OTl4eHgYOEohTIt0UIUQ12nRogUAqamp+Pn5MWzYMOzt7YmMjARg\n7ty5/Pbbb/j7+zNgwAAAPv/8c5ydndFqtUybNs1gsTcEkybFYGk5FVgGLLuauCfG0GGJRuLNN98k\nLi5Ov/3qq68yd+5cJk+ejFarxdnZmS+++AKoOEcMHjxYv+/48eNZtkySTInGqXJ5xtmz97BjRz9C\nQ6NISkoC5LwuxO2QDqoQ4jpVM8Xu3r2buLg48vLyOHz4MNu3b2fChAl06NCB1NRUNm3axG+//ca0\nadPYvHkzu3fvJjMzk6+++sqALTBtQUFBrF1bMf0rIGC9rFMS9Wrs2LF8+umnAJSXl5OYmMj999/P\nnj17yMnJ4b///S+TJ0/mxIkT171XskyLxuzPusr7gTer1VWW87oQt07KzAghbsrT01Nf59PV1RWd\nToe3t3e1fTIzM/H396dNmzYAREREsHXrVh599NF6j7ehCAoKkosXYRCdOnWiTZs27N69mxMnTuDm\n5kZaWhqjR49Go9HQtm1b+vXrR2ZmJi1btjR0uEalsv5x69atadGiBUVFRYYOSRgROa8LcWukgyqE\nuKm77rpL/9jc3JzS0tLr9rm6AF6/LQnQhDBt//d//8fSpUv5/fffGTt2LCkpKdf9XWs0Gpo0aUJ5\nebn+uYsXL9Z3qEal6giyjCY3PlJXWYjaIVN8hRB3xNramnPnzgHQs2dPtmzZwunTpykrK2PlypX4\n+fkZNkAhxB0LDQ1l48aN7Ny5k+DgYHx9fUlMTKS8vJyTJ0+ydetWPD09+cc//kFeXh6XL1+moKCA\nTZs2NZqOWWhoKB4eHjg5ObFo0SJDhyOMgEzjFaJ2yAiqELfo0qVLBAYGkpqa2uAvwG5lFCAmJobg\n4GA6duzIpk2bmDFjBv7+/iileOSRR6olThFCmJamTZvSv39/bGxs0Gg0hIaG8sMPP+Di4oJGo2HW\nrFm0bdsWgOHDh+Pk5IStrS09evQwcOT1Z8mSJdjY2FBSUoKnpyfh4eGGDkkYAZnGK8TfJ3VQhbhF\nS5Ys4fTp00yePNnQoQghRJ0qLy/H3d2d1atX06VLlxr3SUpK0ieAmTQpptFdlE+fPp1169YBcOTI\nETZu3MjIkSPJysqidevWWFtbc/78eQNHKYQQhiV1UIWoQ59//rkk/alBUlISgYHhBAaG69PpC2Hs\nVq1ahYODAwMGDGDLli388MMPhg7JaOTl5fHQQw8xcODAm3ZOQ0OjSEkJISUlpFo5jcagMoN5RkYG\nu3fvxtXVtdGvvxVCiNoiU3yFuAVlZWXs3buXbt26GToUo1J5kVqRVh/S0qJkzY0wCYsXL+Y///kP\n3t7eTJ8+HWtra7y8vG75/aWlpTRp0jB/Qh0cHDh06NBN9/mznEYUACUlFc81lr/9c+fOYWNjg4WF\nBfv37ycjI8PQIQkhRIPRMH9dhahlp06dwtra2tBhGJ3GfpEqTENoaCj/+9//uHjxIs8//zwnTpwg\nPT2dsWPH4uzszLZt2zA3N+ezzz7jo48+olu3bowbN46jR48CMGfOHH1H9tChQ+Tn59OpUydWrFhh\n4JYJQwkODmbBggU4ODhgZ2env7khWXyFEOLvkw6qELdI1kwLYZquTWazZcsWvv/+e2JjY+nRowdv\nvfUW1tbWvPjiiwCMHj2aiRMn4uPjw9GjRwkODiYvLw+An376ibS0tGrll3Q6HYMHDyY3N/eO4jPF\nepmNvZxGs2bN+Pbbb697/vDhw/rHlVnOhRBC3B7poApxC+655x6Tu4CsD439IlWYhri4OH0ym2PH\njnHw4MHr9ql6A+q///0v+/fv12+fP3+e4uJiNBoNISEh1TqntcEUR9oqy2n8mSSp8U7tb+zJooQQ\norZJB1WIW2Bubo6TkxMHDhzAzs7O0OEYDblIFXVh1qxZWFhY8NxzzzFx4kRycnLYtGkT33//PUuW\nLCEqKoo333yTS5cu0aVLF5YuXYqVlVWNx6qazMbCwgJ/f/+/TGajlGLHjh00a9bsuteaN29e43vK\nysqIiYlh+/btdOzYka+++orly5ezaNEiLl++TNeuXVm+fDmWlpbk5+czevRoiouLCQkJuf0vyEhI\nOQ1Zhy+EEHVBsvgKcY0bZaWNiIjQj8KIPwUFBZGcvIbk5DVyUSZqRd++fdm2bRsAO3fupLi4mNLS\nUrZt24azszPvvvsu//3vf8nKysLd3Z0PP/zwhseqmszmp59+qjGZzbXlQAIDA5k7d65+e8+ePX8Z\n88GDBxk/fjx79+6lVatWrFmzhvDwcH788Ud2796Nvb09ixcvBuD555/n2WefJScnhw4dOtzy9yKM\nT/V1+BUd1cobdkIIIe6MdFCFqOJmpRNGjx7NN998Y5JrUQsLC/n4448BOH78OMOGDTNwRELcWI8e\nPcjKyuL8+fNYWFjg5eXFzp07SUtLw9LSkry8PHx8fHBzc+PTTz/VJzOqSXBwMKWlpTg4OPDyyy/X\nmKl38ODBrF27Fjc3N9LT05k7dy47d+7ExcUFR0dHPvnkE/2+N5qOa2tri7OzMwDu7u7odDpyc3Px\n9fXF2dmZFStW6Nexbt++nVGjRgEQGRl5x9+TEEII0RDJFF8hqrhZVtpmzZqxdetWwwZ4h86ePcv8\n+fMZN24c7du3Z9WqVYYOSYgbatq0Kba2tsTHx+Pt7Y2zszPff/89v/zyC7a2tgQEBJCQkHBLx7pR\nMpvNmzfrHz/00EPXjZKuXLlS/7hyVgVUrDGsSdV1qebm5pSUlDBmzBi++uortFoty5YtY8uWLbcU\nszAdsg5fCCFqn4ygCtEITJs2jUOHDuHm5sbw4cPRarUAxMfHM2TIEAIDA7G1teWjjz5i9uzZ9OjR\nAy8vL86ePQvAoUOHGDRoEB4eHvTt25cDBw4YsjmiEfD19WX27Nn069cPX19fFixYQI8ePejduzfp\n6en6Op3FxcU1Jj2qLTebVfFXioqKaNeuHVeuXOGzzz7TP+/j46PvAEupGtNWuQ4/IGA9AQHrZf2p\nEELUAumgClHFpEkxWFpOBZYBy67eDa95xMSUzJw5ky5dupCdnc2sWbOqvbZv3z7Wrl1LZmYmr776\nKi1btmTXrl14eXnx6aefAhATE8O8efPYuXMns2bN4plnnjFEM0Qj4uvry4kTJ/Dy8qJt27ZYWlri\n6+vLPffcQ3x8PKNGjcLFxQVvb+86vWFyq2sMa5r6+/bbb9OrVy/69OmDvb29/vm4uDj+/e9/4+zs\nzG+//WaSWXzFn2Qd/l+bO3cuDg4OPPbYY4YORQhhAmSKrxBVNNSstFXXzV67htbf3x8rKyusrKxo\n1aoVgwcPBkCr1ZKTk0NxcTHbt2+vtm718uXL9RO4aLT69+/PpUuX9NsHDhyoNtX2nXfeMZq/zc6d\nO5OTk6PfnjRpkv7x008/rX9cNf4333xTH/8777xTT5EKU1NYWEhCQgLjxo0zdCh/y8cff8ymTZuq\nJQUrLS2lSRO5DBVCXE/ODKLR0el0DB48mNzc3Bpfb2ylE6qunTMzM9Nvm5mZUVpaSnl5OTY2NmRn\nZxsqRCEMVs6jttYYSjkScSeq5g8wVU8//TSHDx8mODiYo0ePEhISwuHDh+nUqRPvv/8+Y8aM4fTp\n09x7770sXbqUBx54gOjoaJo3b052djZ//PEHixcvZunSpWRmZtKrVy+WLl1q6GYJIeqQTPEVohG4\ntozGragcabW2tsbW1pbVq1frn686WiREfTBUOY/aWmMo5UjEnaiaP2Ds2LFs2LABgNDQUJ544gkA\nlixZwmuvvQbAhx9+iFarRavVEhcXZ7C4q1qwYAEdOnQgNTWViRMnkpeXx6ZNm1ixYgXjx49nzJgx\n7Nmzh4iICCZMmKB/X0FBAT/88AP/+te/CAkJYcqUKezbt4/c3NxbKv0khDBd0kEVjVJZWRkxMTE4\nOTkRFBTExYsXG3QioDZt2uDj44NWq2XKlCn6NW8ajaba+rdrH1dur1ixgsWLF+Pq6oqTkxPr16+v\n3wYIYUCyxlAYStX8AUFBQfr6wL/++iv79+8HYNu2bfTr14+srCzi4+P58ccfycjIYNGiRezevduQ\n4VdTedPz0Ucf1c/UycjIYPTo0UBFyaW0tDSg4vencrmJk5MT7dq1w9HREY1Gg6OjIzqdrv4bIISo\nNzLFVzRKBw8eZOXKlSxcuJARI0awZs0ali5dyoIFC+jatSs7duzgmWeeYdOmTYYOtdbUlC00KiqK\nqKgo/fbhw4evey0pKenqSE9zZs6cKRfowiBMvZyHqccvDKNqzoA+ffowZ84c9u/fj6OjIwUFBZw4\ncYKMjAw++ugj/vOf/xAWFoalpSUAYWFhbNu2DVdXV0OFX6PmzZtX275RbfFmzZoB1ZeeVG6XlpbW\nXYBCCIOTDqpolGxtbXF2dgbA3d0dnU4niYBqIOvmhLEw9QRmph6/MLyOHTtSUFDAxo0b6du3L2fO\nnCExMRFra2usrKzQaDTXJcQz9gzR3t7erFy5ksjISFasWEHfvn0NHZIQwghIB1U0SlXvxpqbm/P7\n77/TqlUrSQR0jerr5qCkpOI5ubAWhmDqCcxMPX5R/67NH9C7d2/mzJnD5s2bOXXqFOHh4QwfPhyo\nKM0UHR3NtGnTKC8vZ926ddXq7xrSjZaSzJs3jzFjxjBr1izatm1bLfnRjd5T07YQomGRDqoQQMuW\nLXnwwQdZvXo1Q4cORSlFbm6ufpRVCCFMRUMpTSKq5w8YNGgQvr6+pKSk8OCDD/LAAw9w9uxZfH19\nAXBzcyM6OhpPT08AnnzySVxcXAwZvl7l8pE333yz2vP/+Mc/alxKU9lRrVxi0q7dQyQlJREUFCQZ\nfIVoBDQ3mvtf5x+s0ShDfbZo3HQ6HSEhIfpMtLGxsRQXF/P4448zbtw4jh8/zpUrVxg1apQ+M2Jj\nde0UX0vLqTLFVwgj91eltIQwBfL7Y3qq3hxLTU0lNjZWn3laNF5Xlx/c1rQH6aCKRu/PJEAViUzk\nx686+X6EMC0jR45k/fr12NnZERAQgFKKjRs3otFoeO211/RTQkXD0tDO1YGB4aSkhFC5xAQqyj0l\nJ68xZFjiJqreHJMOqqh0Jx1UmeIrGjVJAvTXZN2cEKZl5syZ7Nu3j+zsbNasWcMnn3xCTk4OJ0+e\npGfPnvTt25d27doZOkxRi+S3TBiDqnV7mzZtipWVFcOGDWPv3r24u7vr10S/8847bNiwgZKSEry9\nvfnkk08A8PPzo3fv3mzevJmCggIWL15Mnz59DNkkYSBSB1U0atWTAFX8uFfegRZCCFNUdXZSWloa\no0ePRqPR0LZtW/r160dmZqYBoxN1oSH+lk2aFIOl5VRgGbDsammmGEOHJW6iat3eWbNmkZ2dTVxc\nHHl5eRw+fJj09HQAxo8fz48//khubi4lJSV8/fXXQMVIW1lZGTt27GDOnDm89dZbhmyOMCDpoAoh\nhBAN1LWlRyqfE8LYVZZmCghYT0DAehkRNgHXljny9PSkQ4cOaDQaXF1d0el0AHz//ff07t0bZ2dn\nvv/+e/Ly8vTvCwsLA6BHjx76/UXjI1N8RaM2aVIMaWlRlJRUbFfcoV1m2KCEEOJvqFqapE+fPixc\nuJCoqChOnz7N1q1bmT17toEjFLWtof6WyRIT03ZtSb+ysjIuXrzIs88+S1ZWFh07duStt97i4sWL\n173H3Nyc0tLSeo9ZGAcZQRWNmtyhFUI0NFVLk2RkZODs7IyLiwsDBgzQ15sUDYv8lgljcG3d3ppU\ndkbbtGlDUVERq1atqo/QhImREVTR6MkdWiFEQ7NixYpqWV1nz54t57kGTn7LhKFVvTlmaWlZYzK2\nVq1a8eSTT+Lk5ES7du3o1avXDY8nyxEaLykzI4QQQjQwUkNSCGFKGlqZJPGnOykzI1N8xd/SokWL\nWjnOnj17+O6772rlWEIIcTOFhYV8/PHHAKSmpjJ48GADR1T7GmJWVyFEw1R5Qy0lJYSUlBBCQ6NI\nSkoydFjCgKSDKv6W2pp+kZ2dzbffflsrxxKiMWoMna7acvbsWebPn2/oMIQQQiA31MT1pIMqbllo\naCgeHh44OTmxaNEi/fMvvvgiTk5ODBw4kFOnTgGwe/duevfujYuLC2FhYRQUFAAVRZizsrIAOHXq\nFLa2tly5coU33niDxMRE3NzcZMG8uI6Pj4+hQzB60um6dVWLyU+ZMoWioiKGDRuGvb09kZGR+v2y\nsrLw8/PDw8OD4OBgTpw4AVScx1588UV69uyJvb09mZmZhIaG0q1bN15//XVDNasaqSEphBDCZCml\nDPKv4qOFKTlz5oxSSqkLFy4oJycndfr0aaXRaFRCQoJSSqm3335bjR8/XimllFarVVu3blVKKfXG\nG2+oF154QSmllJ+fn8rKylJKKXXy5EnVuXNnpZRS8fHx6rnnnqvX9gjRkIwYMUJZWloqV1dX1bNn\nT+Xn56eGDh2qunfvriIiIvT77dy5U/Xr10+5u7uroKAgdfz4caWUUnFxccrBwUE5OzurkSNHKqWU\nKioqUmPGjFGenp7Kzc1NffXVVwZpW23T6XTKyclJKaVUamqquvvuu9Wvv/6qysvLlZeXl0pLS1OX\nL19WXl5e6tSpU0oppVauXKnGjh2rlKo4j02bNk0pVfG9tW/fXp04cUJdunRJ3X///fpzpaFt3LhR\nBQSEqYCAMLVx40ZDhyOEEDXauHGjsrS8T0G8gnhlaXmfnLMakKt9vtvqJ0oWX3HL4uLiWLduHQDH\njh3j4MGDmJmZMWLECAAiIyMJCwvj3LlzFBYW4uvrC0BUVBTDhg276bHVnzcuhLhOixYtKCoqMnQY\nRm3mzJns27eP7OxstmzZwqOPPkpeXh7t27fHx8eH9PR0PD09ee6559iwYQNt2rQhMTGRV199lcWL\nFzNz5kx0Oh1Nmzbl3LlzALz33nsMGDCAJUuWUFBQQK9evRg4cCDNmzc3cGv/nqrnGlWlmDygLyZ/\n9913s2/fPgYOHAhAWVmZfh+AkJAQAJycnHBycuK+++4D4MEHH+To0aPY2NjUV3NuSLK6CiFMQWWZ\npD+TJElCt8ZOOqjilqSmprJp0yYyMjKwsLDA399fX8uq8mJPKVXjmtSqF4NNmjShvLwcoFphZiFu\nRlLN/7W/2+lydnZm9OjRDBkyhCFDhgCQnJzMhg0bmD17NgCXLl3if//7H3Z2dvXZtDp3bTH5yuLw\njo6ObN++/abvMTMzq/Z+MzMzysrK6jBaIYRoeOSGmqhK1qCKW3Lu3DlsbGywsLBg//79ZGRkAFBe\nXs7q1asBSEhIwNfXl5YtW2JjY0NaWhoAy5cvx8/PD4DOnTuzc+dOAP37AFq2bPmXxZ2FELfuZp2u\n7OxssrOzycnJYePGjQB88803PPvss+zatYuePXvqO1lffvmlfn+dTtcgOqd/VUxeo9FgZ2fHyZMn\n9ee6K1eukJeXV18hCiGEEI2WdFDFLQkODqa0tBQHBwdeeeUVvLy8ALCysuLHH39Eq9WSmprKG2+8\nAcCyZcuYPHkyLi4u5OTk6J9/6aWX+Pjjj+nRowenT5/Wj4z5+/uTl5cnSZKEuEN/p9OllOLo0aP4\n+fkxY8YMCgsLKSoqIigoiLlz5+qPkZ2dXeftqA9Vi8lPmTKlxhH6pk2bsnr1aqZOnYqrqytubm78\n8MMP1+2n0WhkhF8IIYSoRRpDrfvTaDRK1hw2bg2pKPOyZcsIDAykffv2hg6lQfqrzpeoEBERQU5O\nDpaWlrRr147169cD8Nxzz9GzZ08ef/xx9uzZw4QJEygsLKS0tJSJEycSFRWFv78/hYWFKKV47LHH\nmDJlChcvXuSFF15g+/btlJeX8+CDD+qPKYQQQgjxVzQaDUqp27qTKx1UYRCVRZkr6l6BpeVU1q41\n3UXx/v7+zJ49G3d3d0OH0iC1bNlSn7hH1I+GdAOprsh3JIQQQtzcnXRQZYqvMAhTKMr84YcfotVq\n0Wq1xMXFodPp0Gq1+tdnz57NW2+9xZo1a9i5cycRERH06NFDkj/VAemc1q/KG0gpKSGkpIQQGhpF\nUlKSocMyKvIdgU6no3v37owZMwY7OzsiIiJITk7Gx8eHbt26kZmZSWZmJt7e3vTo0QMfHx9+/vln\nAOLj4wkLC2PQoEF069aNqVOnArBkyRImTpyo/4xFixbx4osvGqR9QgghDEM6qELUICsri/j4eH78\n8UcyMjJYtGgRBQUF1fapXHsWHh6Oh4cHCQkJ7Nq1CwsLCwNFbfqSkpIIDAwnMDC80V3sGxNTuIFk\naPIdVTh06BAvvfQSP/30EwcOHCAxMZH09HRmz57N+++/j729Pdu2bWPXrl289dZbvPLKK/r37tmz\nhy+++ILc3FwSExP59ddfGTFiBBs2bNAn6YqPj+eJJ54wVPOEEEIYgJSZaWT8/PyIjY2ts6mo0dHR\nDB48mPDw8JvuN2lSDGlpUZSUVGxbWk5l0qRldRLTnUhLSyMsLAxLS0sAwsLC2Lp163X7XVvaQ9y5\na6d9p6VFmfS0byEaA1tbWxwdHYGKDNGVJYycnJzQ6XQUFBTw2GOP8csvv6DRaPTZpAEGDBiAtbU1\nAA4ODhw5coSOHTvSv39/NmzYQPfu3bly5Yr++EIIIRoH6aA2MnWdcfJWj2/sRZmvzpfXbyulKCws\n1NdwBSgpKanWVsnk+fdUH5GCkpKK54zp/4vGwthvIBkD+Y4qXFsDtlmzZvrHpaWlvP766wwYMIC1\na9dy5MgRfcmxa99btRTS//3f//Hee+9hb2/P2LFj66chQgghjIZM8W2gKtcGRUZG4uDgwLBhwyip\nvJK66plnnqFnz544OTkxffp0/fOZmZn4+Pjg6upKr169KC4upqysjMmTJ+Pp6YmLiwsLF1Z0LJVS\njB8/nu7duxMQEMAff/xxyyOJQUFBJCevITl5jdF1Qnx9fVm3bh0lJSUUFxezbt06Bg0axB9//MGZ\nM2e4dOkSX3/9tX5/a2trWScpGozKG0gBAesJCFgvI9k1kO/orymlOHfuHB06dABg6dKlf7k/gKen\nJ8eOHSMhIYFRo0bVeZxCCCGMi4ygNmA///wzS5cuxcvLiyeeeIL58+dXe/29997DxsaGsrIyBg4c\nSG5uLnZ2dowcOZIvvvgCd3d3ioqKsLCwYPHixbRq1Yoff/yRS5cu0adPHwIDA9m1axdGuJDtAAAg\nAElEQVQ///wz+/fv58SJEzg4ODSI9UJubm5ER0fj6ekJwJNPPomHhwdvvPEGnp6edOzYEQcHB/3+\n0dHRPP300zRv3pzt27fLOtQ7ICNSxiUoKEg6XH9BvqPrZ45U3TYzM2Py5MlERUXx7rvv8vDDD+tf\nr2m2TdXt4cOHs2fPHu6+++46jF4IIYQxkjIzDZROp6Nfv34cOXIEgM2bNzN37lwKCgqIjY2lR48e\nLFiwgEWLFlFaWsrx48eZN28eDg4OjBs3jrS0tGrHGzp0KLm5uTRv3hyoyKq6YMECvv32W1xcXIiO\njgYgPDyciIgIwsLC6rW9hiAlJmqffKdCNF5V//7PnfuVDz74AH9/fwNHJUzZrebFEELUnTspMyMj\nqA1Y1bvRSqlq2/n5+cTGxrJz507uvvtuxowZw8WLF2+6jvKjjz4iICCg2nPffvtto0wOJAl96oaM\nSAnROP15Tn0DeAdz87NcvnzZ0GEJE1fXeTeEEHVD1qA2YEePHiUjIwOAhIQE+vTpA/y5LsjKyoqW\nLVvy+++/891336HRaLCzs+P48ePs3LkTgPPnz1NWVkZQUBDz58/XJ7H4+eefuXDhAn379iUxMZHy\n8nKOHz/O5s2bDdPYeiYlJoQQovb8eU59BjhOWdknck4VNbq2RvmRI0ewt7cnJiYGJycngoKCqtUj\nV0qxefNmQkP/f3v3HldVmf7//30rmDtHjcE+peYvUZEEQRDDUCicQmxKHXLIShpEzcmOH6PU6TBW\nE02O0ow2qU2jZpiOjo6Z9bENg2GRWYkYGGppv60dzA6aB8IDsr5/gASKJgh7bdiv5+Pho73XYe9r\ntVzuda37vq87sWpZdna2V/T0ApoqEtRmLCgoSM8//7yCg4N14MABTZgwQVLFE8U+ffooIiJCV1xx\nhUaNGlWVvPr6+mrp0qW69957FR4eroSEBB09elTjxo1TcHCw+vbtq9DQUE2YMEEnTpxQYmKiAgMD\nFRwcrJSUFA0YMMDOQwYAAM1UbXOU79+/Xzt27NA999yjLVu26KKLLtKKFSuq9jHGaNCgQdq2bZu+\n//57SRUFu5pDvQxvcLLoZ2pqqoKCgjRq1ChlZWVpwIAB6tmzpz744AP17NlT3333nSSpvLxcgYGB\nVecaTRNdfJsxHx8fZWZm1lhWvYXzTBUV+/Xrp/fee++05enp6UpPTz9t+XPPPXeekTY9FPQBgIbD\nv6k4F7XNUf7OO+8oICBAYWFhkqTIyEi5XK7T9r399tuVmZmp0aNHa8OGDVq0aJE7Q8d52Llzp1as\nWKHg4GBdeeWVWrp0qdavX6/XXntNf/7zn3X77bfrlVde0f3336///ve/Cg8Pl7+/v91h4zzQgtqM\nNea4C6fTqcGDR2jw4BFyOp2N9j2eiikmAKDh8G8qzsWpc5SfdKY5daWfpi9KTU3VokWL9K9//Us3\n33yzWrTgFripCAgIUEhIiIwxCgkJ0XXXXSdJ6t27t1wul1JTU/Xyyy9LkubPn6/U1FQ7w0UDoAW1\nmeratasKCwsb5bMpEFSBgj4A0HD4NxU/JzY2VqNHj9aUKVNUXl6ulStXKjMzs2pu9rPp2LGjOnXq\npKeeeko5OTluiBYNpfoDiBYtWqhVq1ZVr8vKynTZZZfpkksu0dq1a/Xhhx9qyZIldoWKBkKCijqr\nWSBIKi2tWMaNBQAAaCy1zVHu5+d31jl1q7++7bbb9N133ykoKMg9AcNtxo0bp+TkZKWkpFC5uRkg\nQQUAAECTMHHiRE2cOLHGsuo9xtLS0qpeL1iwoGpIkiS1bn1cd9xxh3sCRYM5lwcQQ4cOVWpqKt17\nmwk64KPO0tLGy+GYLGmhpIWVxSzG2x0WAMDNDhw4oDlz5kiScnNzNXTo0Fq3u+OOO7R161Z3hgZU\nDUnKzh6m7Ox8vfHGm/qf//kfu8NCHZw6ZG3BggVVUwR17dpV06dP1+DBIxQTk6DLL79cPXv2tCtU\nNCASVNQZxSwAAJK0f/9+zZ49+2e3e/HFF9WrVy83RAT8pOaQJJfKy1/UzJnza2zjcrkUGhp63t/V\ntWtX7du377w/B+fupwcQLfTBB4Xavv0Lryzc2RyRoKJeEhISlJW1QllZK0hOAcBLTZkyRTt37lRE\nRIQmTZqkw4cPKykpSb169VJycnLVdnFxcdq0aZPKy8s1evRohYaGKiwsTH/7299sjB5oOIx7dL+f\nHkD8W9J3Onbsr8rI+PmCWfB8JKgAAKBepk2bpu7du6ugoEDTp09XQUGBZs6cqeLiYn322Wdav369\npJ9u3gsKCvTVV1+pqKhIhYWFjBdDozrXIUllZWVKTk5WcHCwkpKSVFpaqpycHPXt21dhYWEaO3as\njh07JklnXH5SaWmprr/+es2bN88NRwg70Fre+EhQAXis8vJyu0MAcBbV56S0LEtRUVHq1KmTjDEK\nDw+Xy+WqsX337t312Wef6b777pPT6VS7du3cHDG8ybkOSdq+fbvuvvtuFRcXq127dsrIyFBqaqqW\nLVumwsJClZWVac6cOTpy5Eity086dOiQhg0bplGjRmns2LHuPFSvZFdNFFrLG1+jJKjGmMeNMV8Y\nYwoq/wxpjO8B4DmmTp2qmTNnVr1/5JFHNGvWLE2fPl1RUVHq06ePHn/88ar1iYmJ6tevn3r37q0X\nX3yxavkvfvELPfjggwoPD9d7772nKVOmKCQkRH369NFDDz3kzkNCE7Ju3Tq99957dofh9arPV9iy\nZUuVlZXVWH/RRRfpo48+UlxcnObOnatx48a5O0R4mXMZktSlSxdFR0dLkpKTk7V27Vp169ZNPXr0\nkCSlpKTo7bff1ieffKKAgIDTlksVD2iGDx+uMWPG1OjejsbjjpooZ7pXkaSSkhLdcMMNCg8PV2ho\nqJYtWybp51vZ8fMaa5oZS9KzlmU920ifD8DDjBkzRjfddJPuv/9+lZeXa+nSpXr66aeVk5OjDz74\nQOXl5Ro+fLjeeecdxcbGav78+fLz81NpaamioqL029/+Vn5+fvrxxx911VVXacaMGfr+++81duxY\nbdu2TZJ08OBBm48Snuqtt95S27Ztq24y4R5t27bVoUOHzmlby7L0/fffy9fXVzfddJN69uyp22+/\nvZEjBH5e9RYxy7J00UUX6fvvv6+xrDbVlxtjFBMTozVr1ujWW29tvGBRQ0JCQqPWQjn1XmXEiIop\niyzL0ptvvqnOnTvrjTfekFRxj3KylX3t2rXq0aOHUlJS1LFjxxp/n/DzGrOLL+3fgBe5/PLL5e/v\nr82bNysrK0sRERH68MMPq15HRkZq+/bt2rFjhyRp5syZCg8PV3R0tD7//HN9+umnkipaXU7+ALRv\n316tW7fW2LFjtXLlSjkcDtuOD+7lcrl0xRVX1BgX9uOPP9YY+7Nx40YNGjRIu3bt0gsvvKC//vWv\nioiIUF5ens3Rew9/f38NHDhQoaGhmjRp0lm7vhlj9OWXX2rQoEGKiIjQ7bffrmeeecaN0QK12717\ntzZs2CBJWrx4sfr16yeXy6WdO3dKkjIzMxUXF6egoKBal5/05JNPys/PT3fffbfbjwGNo/q9yhdf\nfFF1r2KMUVhYmLKzszVlyhTl5eWpXbt22r59+2mt7EeOHLHzEJqkxmpBlaR7jTG/k7RRUpplWT80\n4ncB8ADjxo3TggULtHfvXo0ZM0Y5OTn6wx/+oPHja44Jyc3NVU5OjjZs2KDWrVtr0KBBVf+At27d\nuuom18fHRx988IFycnK0fPly/f3vf1dOTo7bjwv2+OSTT7RgwQJFR0dr7Nixmj17dq0J0OWXX647\n77xTbdu21QMPPGBDpN7tlVdeqXX5c889V/X6rbfeqnqdn5/f6DEB58oYo6CgID3//PMaM2aMQkJC\n9MADD+iqq65SUlKSysrKFBUVpTvvvFO+vr5asGDBactPfo5UkdCMGTNGkydP1rRp0+w8NJyns92r\nSFJgYKAKCgr0xhtv6NFHH9W1116r4cOH1/gMy7Kqhj7s2bNHI0eO1KFDh6rGL8fExLj1mJqKeieo\nxphsSZfWsuoRSXMkPVn5/k+SMiSdNlq8+ni0uLi4Gk+hADQ9iYmJeuyxx3TixAktWbJEPj4+euyx\nxzRq1Ci1adNGX375pVq1aqWDBw/Kz89PrVu31rZt26qeXJ+qpKREJSUluv766zVgwAB1797dzUfk\nPuXl5WrRgrp11Z06Lqz6GOfanKkbHjyD0+msmgIiLW08U5TBI1x++eXaunXract/9atfadOmTee8\n/LPPPqt6PX/+/NPWo+mpfq+ydevW0+5V9uzZIz8/P40aNUrt27fX/PnzNWnSpKpW9u7duyszM1NP\nPPGEJGnJkiUaMmSIHn74YVmWpZKSEjsOq9Hl5uYqNzf3vD6j3gmqZVnx57KdMeafklbXtq56ggqg\n6fP19dWvfvUr+fn5yRij+Ph4bd26tSrJaNu2rRYtWqQhQ4Zo7ty5Cg4OVlBQUI1xg9VbyA4dOqTh\nw4fryJEjsixLf/3rX91+TOdi+vTpat26te69915NnDhRhYWFysnJ0dq1azV//nylpKRo6tSpOnr0\nqLp3764FCxaoTZs26tq1q2655RZlZ2dr0qRJ8vPz0+OPP37adt7q1HFhLVq0kI+PT1V1Z7pNNR1O\np1OJiSmVcxZKeXkpjVLQBLADD1+apzPdq5z8bSoqKtJDDz2kFi1ayNfXV3PnztUFF1xwxlb2K6+8\nUmPGjNHx48f1m9/8Rn369LHt2BrTqY2OJxP0ujCN8cTZGNPRsqw9la8nSrrSsqzbTtnG4mk30LyU\nl5crMjJSy5cvb9atnad6//33lZGRoWXLlik2NlbHjx9XXl6e0tPT5XA49MYbb2jNmjW68MILNW3a\nNB07dkyPPfaYAgICdPfdd+vBBx/Ud999pxEjRujNN9+Uw+GosZ03crlc6tatm9avX6+rrrpK48aN\nU3BwsNasWaO0tDQNGTJEEydO1ObNm/XWW2/p2Wef1cGDB3nw6aEGDx6h7OxhklIql1RU3szKWmFn\nWMB5O/Xhi8MxmYcvOKOvv/5ar7/+up5//nk98MADXlEozhgjy7LqVJuosfqTTTPGFBpjPpJ0jaSJ\njfQ9ADxEcXGxAgMDdd111513cup0OjV48AgNHjxCTqezgSJsPH379lV+fr4OHTqk1q1bKzo6Whs3\nblReXp4cDoeKi4s1cOBARURE6OWXX9bu3bur9h05cqQkacOGDSouLtaAAQNq3c4bnRwXFhwcrAMH\nDuiuu+7S1KlTdf/99+vKK6+Uj49P1ZPsoUOHauXKlYqIiNC7775rc+QAvEVGxj8qk9MUSRWJ6snW\nVHiXn7t32b17ty6++GKNGzdO48aNU0FBgQ1RNg2NUiTJsqzfNcbnAvBcwcHBVZUNz0dT7Aro6+ur\ngIAAvfTSSxowYIDCwsK0du1a7dixQwEBAYqPj9fixYtr3bd6F96zbeeNfHx8lJmZWWNZTEyMtm/f\nftq2gYGB+uijj9wVGuooLW288vJSVFpa8d7hmKy0tIX2BgUADeRc7l1yc3M1ffp0+fr6qm3btnr5\n5ZftCtfjUZEDgEdpqk+jY2NjNWPGDF1zzTWKjY3V3Llz1bdvX1111VV69913q5L3kpKSqjL11fXv\n3/+ctvMmZ5uypKm1snu7hIQErVxZ0a03Pv41j3/oBJyrtLTxcjgmS1ooaWHlw5fxP7cbmpmz3buc\n/L1atGiVZsyYoU2bNmndunW6/PLLbY3ZkzXmNDMA4DViY2P19NNPKzo6Wg6HQw6HQ7GxserQoYNe\neukl3XrrrTp69KgkKT09XYGBgTX2v/jii89pO2/RtWtXFRYW1rquKbayoyJJ5RyhuTn58OWnIkn8\nW4Sf8HtVP41SJOmcvpgiSQBqQcEJ/BwK7gD2cblcGjp0qIqKiuwOBfAYZ7p3ycj4h9f/XnlSkSQA\nqBdv7ApId1UA3uDEiRN2hwA0Cm+8d2lMtKACgI1oMa47/p8B9nG5XPr1r3+tmJgYrV+/Xp07d9aq\nVav05Zdf6p577tG3336rCy+8UC+++KKCgoI0evRotW7dWps3b1ZMTIxmzJhh9yEAbsPvVf1aUElQ\nAcBGdFetH6fTWW3M13iv+rEH7ORyuRQYGKj8/HyFhYVp5MiRGjZsmBYsWKC5c+eqR48eev/99/Xw\nww8rJydHo0eP1r59+7Rq1aqzFj4Dmitv/72qT4JKkSQAQJNDwR3APgEBAQoLC5MkRUZGyuVyaf36\n9UpKSqra5tixY5Iqbk6TkpJITuG1+L2qOxJUALAR80MCaGouuOCCqtctW7bU3r17ddFFF6mgoKDW\n7S+88EJ3hQagGaBIEgDYiMIKAJq6du3aqVu3blq+fLkkybKsM04TBQA/hxZUALAZ3X8ANCWndtc1\nxmjRokWaMGGCnnrqKR0/fly33nprVTdguvcCqAuKJAEAAKDBeHtRGAA/oYovAAAAbMO0GgCqI0EF\nAACAbZg6C0B19UlQKZIEAAAAAPAIJKgAAABoEGlp4+VwTJa0UNLCyqmzxtsWz6xZsxQcHKxf/vKX\n+stf/nLO++3atUtLlixpxMgAnAldfAEAANBgPKlIUq9evZSTk6NOnTrVuv7EiRNq2bLlactzc3OV\nkZGh1atXN3aIQLPGGFQAAABA0p133qkFCxYoKChIY8aM0c6dO/Xcc89p9OjRat26tTZv3qyBAwdq\n2LBh+t///V9JUosWLbRu3Tpdd9112rZtmwICAjR69Gjdf//9Nh8N0DTVJ0FlHlQAAAA0O3PnzpXT\n6VRubu5pLaFfffWV3nvvPRljNGzYMM2ePVvR0dH68ccfdcEFF2jatGmaMWMGLaiADRiDCgAAgGbL\nsixV77VnjFFSUpKMqWjUGThwoCZOnKjnnntO+/fvV8uWLUUvP8A+JKgAAADwKhdeeGHV68mTJ2ve\nvHkqLS3VwIEDtX37dhsjA0AXXwAAAHitnTt3KiQkRCEhIfrwww+1fft2XXbZZTp06JDdoQFeiRZU\nAAAANEvGmBp/qi8/aebMmQoNDVWfPn3UqlUrXX/99QoLC1PLli0VHh6umTNn2hE64LWo4gsAAACv\n50nT4wDNBdPMAAAAAHXkdDqVmJii0tJpkiSHY7JWrlxIkgqcp/okqHTxBQAAgFfLyPhHZXKaIqki\nUT3ZmtoYZs+erYiICPXt21dff/11o30P0BSRoAIA4EYul0uhoaF2hwHARnfddZcKCgq0adMmXXrp\npXaHA3gUqvgCAADAq6WljVdeXopKSyveOxyTlZa20N6gAC9FCyoAAG5WVlam5ORkBQcHKykpSaWl\npcrPz1dcXJz69eunIUOG0O0PcKOEhAStXLlQ8fGvKT7+tQYff+p0OjV48AgNHjxCTqezwT4XaI4o\nkgQAgBu5XC5169ZN7777rqKjozV27FhdccUVevXVV7Vq1Sp16NBBS5cuVVZWlubNm2d3uADOEwWY\n4M3qUySJLr4AALhZly5dFB0dLUlKTk5Wenq6tmzZovj4eEnSiRMn1KlTJztDBNBAahZgkkpLK5aR\noAK1I0EFAMDNjPnpYbJlWWrXrp1CQkK0fv16G6MCAMB+jEEFAMDNdu/erQ0bNkiSFi9erKuuukrf\nfvtt1bLjx4+ruLjYzhABNJC0tPFyOCZLWihpYWUBpvF2hwV4LMagAgDgRrt27dKQIUPUr18/5efn\nKyQkRJmZmdq+fbvuu+8+HThwQGVlZZo4caLGjh1rd7gAGoDT6ayaVzUtbTzde+E16jMGlQQVAAAA\nANDg6pOg0sUXAAAbMf0EAAA/oQUVAACbMP0EAKA5o4svAABNyODBI5SdPUwnp5+QFio+/jVlZa2w\nMywAABoEXXwBAABstmrVKm3dutXuMACgSSJBBQDAJkw/0fyUlZVp5cqVTBMEAPVEF18AAGzE9BOe\nx+VyVU0FtGnTJoWEhOjll1/W9OnT9frrr6u0tFQDBgzQCy+8IEmKi4tTRESE8vLylJiYqIyMDLVv\n317t27fXihUr1K1bN5uPCADswRhUAACA8+RyudStWze9++67io6O1tixYxUcHKwxY8bIz89PkvS7\n3/1ON998s2688UYNGjRIISEh+vvf/y5JSk1N1dChQ3XTTTfZeRgAYDvGoAIAADSALl26KDo6WpKU\nnJysvLw8rV27Vv3791dYWJjWrl1boxvvyJEja+zPQ3gAqB8fuwMAAADwNMb89MDfsiwZY3T33Xcr\nPz9fnTt31hNPPKEjR45UbdOmTZsz7g8AOHe0oAIAAJxi9+7d2rBhgyRp8eLFiomJkST5+/vr8OHD\n+ve//11j++otpm3bttXBgwdP+0yXy6XQ0NBGjBoAmj4SVAAAgFMEBQXp+eefV3BwsA4cOKAJEybo\njjvuUO/evTVkyBD179+/xvbVW0xvueUWTZ8+XZGRkfrss8/cHToANGkUSQIAAKjG5XJp6NChKioq\nOqftz7USs8vl0q9//WvFxMRo/fr16ty5s1atWqXMzEy9+OKLOnbsmHr06KHMzEw5HA79+9//1pNP\nPqmWLVuqffv2WrduXYMdIwC4A0WSAAAAGsC5jiF1Op1KTExRdvYwZWcPU2JiipxO5xm3//TTT3XP\nPfdoy5Ytuuiii7RixQqNGDFCH3zwgTZv3qxevXpp3rx5kqQ//elPysrK0ubNm7V69eoGOS4A8HQk\nqAAAANV07dpVhYWF57RtRsY/VFo6TVKKpBSVlk6rak2tTUBAgMLCwiRJkZGRcrlcKioqUmxsrMLC\nwvTKK69UVQceOHCgUlJS9M9//lNlZWXne1gA0CSQoAIAgCbjwIEDmjNnjiQpNzdXQ4cOrdP+U6dO\nVU5OTmOEdk4uuOCCqtctW7ZUWVmZUlNTNXv2bBUWFmrq1KkqLS2VJM2ZM0dPPfWUPv/8c0VGRmrf\nvn12hQ0AbkOCCgAAmoz9+/dr9uzZ9d7/iSee0LXXXnva8vLy8np9XlraeDkckyUtlLRQDsdkpaWN\nr9NnHD58WJdeeqmOHz+uRYsWVS3fuXOnoqKi9MQTT+jiiy/WF198Ua8YAaApIUEFAABNxpQpU7Rz\n505FRERo0qRJOnz4sJKSktSrVy8lJydXbZefn6+4uDj169dPQ4YM0ddffy1JGj16tFasWCGpoivv\nlClTFBkZqeXLl9crnoSEBK1cuVDx8a8pPv41rVy58IxFkqTax7Y++eST6t+/v2JiYtSrV6+qbSZN\nmqSwsDCFhoZq4MCBVV2DAaA5o4ovAABoMnbt2qUbb7xRRUVFWrdunYYPH67i4mJ17NhRAwcO1PTp\n0xUVFaVrrrlGq1evlr+/v5YuXaqsrCzNmzdPqampGjp0qG666SYFBATo7rvv1oMPPmj3YdVwrlWB\nAcDT1aeKr09jBQMAANDQqj/ctixLUVFR6tSpkyQpPDxcLpdL7du318cff6zrrrtOknTixImqbU41\ncuTIxg+6Dk5WBa4ovCTl5aX8bKssADQnJKgAAKDJqq3okCSFhIRo/fr1P7t/mzZtGi22+qhZFVgq\nLa1YRoIKwFswBhUAADQZbdu21aFDh8643hijoKAgffvtt9qwYYMk6fjx41VTtwAAPBstqAAAoMnw\n9/fXwIEDFRoaKofDoUsvvfS0bXx9fbV8+XLdd999OnDggMrKyjRx4kQFBwfbEHHdpKWNV15eiipn\nmqmsCrzQ3qAAwI0okgQAALxGUyhA1BRiBIBzUZ8iSSSoAADAK5xagMjhmEwBIgBoRCSoAAAAZzB4\n8AhlZw/TyQJEUsX8pVlZK+wMCwCarfokqBRJAgAAAAB4BIokAQAAr0ABIgDwfHTxBQAAXoMCRADg\nPoxBBQAAAAB4BMagAgAAAACaLBJUAAAAAIBHIEEFAAAAAHgEElQAAAAAgEcgQQUAAAAAeAQSVAAA\nAMALxcXFKT8/3+4wgBpIUAEAAAAvZIyRMXWaAQRodD52BwAAAADg7Fwul4YOHaqioiJJ0owZM1RS\nUiI/Pz+98MIL8vHxUXBwsJYsWaKSkhLde++9+vjjj3X8+HE9/vjjGjZsmEpLS5WamqrCwkJdccUV\nKi0tlWVZNh8ZUBMJKgAAANDEnGz5nDZtmlwul3x9fXXw4EFJUnp6uq699lrNnz9fP/zwg/r376/r\nrrtOc+fO1S9+8QsVFxerqKhIffv2pQUVHocuvgAAAEATFRYWpttuu02vvPKKWrZsKUnKysrSM888\no4iICA0aNEhHjx7V7t279c477yg5OVmSFBoaqrCwMDtDB2pFCyoAAADg4Xx8fFReXl71vrS0VJL0\nf//3f1q3bp1Wr16t9PT0qi7A//nPfxQYGHja59ClF56OFlQAAADAw11yySX65ptvtG/fPh09elSv\nv/66ysvLtXv3bsXFxemZZ57RgQMHdPjwYSUkJGjWrFlV+xYUFEiSrr76ai1evFiStGXLFhUWFtpy\nLMDZ1DtBNcYkGWM+NsacMMb0PWXdH4wxnxpjthljBp9/mAAAAPBGf/vb36paC72Zr6+v/vjHPyoq\nKkqDBw9WcHCwTpw4oeTkZIWFhalv3766//771b59ez322GM6fvy4wsLC1Lt3b02dOlWSNGHCBB0+\nfFjBwcGaOnWq+vXrZ/NRAacz9W3mN8ZcIalc0guS0izL2lS5PFjSYklXSuos6b+SelqWVX7K/hZd\nDAAAAHA2AQEB2rhxo/z9/c95n/LycrVoQUfBk5xOpzIy/iFJSksbr4SEBJsjgrcwxsiyrDpV4qr3\nlWtZ1jbLsj6pZdVwSUssyzpuWZZL0g5JUfX9HgAAADR906dP13PPPSdJmjhxoq699lpJ0tq1a5Wc\nnKy77rpLV155pXr37q3HH39ckjRr1ix99dVXGjRoUNX2WVlZGjBggCIjI3XzzTerpKREktS1a1dN\nmTJFkZGRWr58ufsP0EM5nU4lJqYoO3uYsrOHKTExRU6n0+6wgDNqjEdLnSR9UUknPJYAAA9USURB\nVO39F6poSQUAAICXuvrqq/XOO+9IkjZu3KiSkhKVlZXpnXfe0TXXXKP09HR9+OGH+uijj7Ru3Tpt\n2bJF9913nzp16qTc3Fzl5OTou+++U3p6unJycpSfn6/IyEg9++yzkipaajp06KD8/HzdfPPNdh6q\nR8nI+IdKS6dJSpGUotLSaVWtqYAnOmsVX2NMtqRLa1n1sGVZq+vwPbX25T35dEyS4uLiFBcXV4eP\nBAAAQFPRt29f5efn69ChQ2rdurX69eunjRs3Ki8vT7NmzdLSpUv14osvqqysTHv27FFxcbF69+5d\n4zM2bNig4uJiDRgwQJJ07NixqteSNHLkSLceE4CacnNzlZube16fcdYE1bKs+Hp85peSulR7f1nl\nstNUT1ABAD85cOCAFi9erAkTJig3N1cZGRlavbouzwUBwLP4+voqICBAL730kgYMGKCwsDCtXbtW\nO3bskMPhUEZGhjZu3Kj27dsrNTVVR44cqfVz4uPjqyrRnqpNmzaNeQhNUlraeOXlpehknSmHY7LS\n0hbaGxSarVMbHZ944ok6f0ZDdfGtPvD1NUm3GGNaGWMCJAVK+qCBvgcAvML+/fs1e/bsBvmsEydO\nNMjnAMD5io2N1YwZM3TNNdcoNjZWc+fOVd++fXXw4EG1adNG7dq10969e7VmzZqqfdq2bauDBw9K\nkvr37693331XO3fulCSVlJTo008/teVYmoqEhAStXLlQ8fGvKT7+Na1cuZAiSfBoZ21BPRtjTKKk\nWZI6SHrDGFNgWdb1lmUVG2OWSSqWVCbpLsr1AkDdTJkyRTt37lRERIR8fX3Vpk0bJSUlacuWLXI4\nHEpNTdW9996r2267TWvWrFH37t3VokULdenSRZdccokyMzPVqlUrtW7dWg8++KCuvvpqJSYm6ttv\nv5WPj49uv/32BkuAAeBcxcbG6umnn1Z0dLQcDoccDodiY2MVFhamiIgIXXHFFerSpYtiYmKq9hk/\nfryGDBmizp07KycnRy+99JJuvfVWHT16VJKUnp6uwMBAuw6pSUhISCApRZNR72lmzvuLmWYGAM5o\n165duvHGG1VUVKR169Zp+PDhKi4uVseOHdWnTx916NBBTqdT/v7+CgwM1Pvvv69bb71Ve/bs0erV\nq3XTTTcpODhYxcXFevbZZ/X73/9e+/fv16effqqlS5fq9ddfV2Zmpt2HCQAAmrH6TDNT7xZUAEDj\nqf4Az7IsRUVFqVOnTpKkAQMG6D//+Y82bdqkH3/8UZ9//rmCg4O1Z88ehYeHa+nSpdq4caM+//xz\nHTx4UDk5Odq+fbuOHTsmf39/tW3bVkFBQXYdGgA0GOb3BJofZjAGgCbgggsuqHrt6+uriy++WK++\n+qo6duyoOXPmaPTo0erQoYMyMzOVkZGh8PBwLVu2TDfccIOOHDmikJAQHThwQIsWLdK1116rsrIy\nG48GAM4f83sCzRMJKgB4oLZt2+rQoUNnXN+zZ0+98sorsixLDodDc+fOVXh4uAoKCtSmTRv5+Pjo\nu+++05o1a9SpUyd98803+u9//6vrr79e06ZNU35+vhuPBgAaHvN7As0TCSoAeCB/f38NHDhQoaGh\nmjRpkoypOXwjKChIe/fu1YoVKzRt2jR988032rBhg/bt26eIiAi9//77evTRRxUTEyMfHx+98MIL\nuuWWW+RwOHTZZZfpt7/9rU1HBgAAcGYUSQIAAECTc7KLb0UrasX8nkyhAniW+hRJogUVAJqx9PR0\n+fv3kL9/D6Wnp9sdDgA0GOb3BJonWlABoJlKT0/Xo4/+RRVTVkvSfXrqqUl65JFH7AwLAAB4ifq0\noJKgAkAz5e/fQ/v2PaaKAiKStFC//OWf9P33O+wMCwAAeAm6+AIAAAAAmiwSVABoph54IFXSfZIW\nVv65r3IZAACAZ6KLLwA0Y+np6Xr22QWSKhJWxp8CAAB3YQwqAAAAAMAjMAYVAAAAANBkkaACAIA6\nKykp0Q033KDw8HCFhoZq2bJlysnJUd++fRUWFqaxY8fq2LFjdocJAGhiSFABAECdvfnmm+rcubM2\nb96soqIiJSQkKDU1VcuWLVNhYaHKyso0Z84cu8MEADQxJKgAAKDOwsLClJ2drSlTpigvL08ul0sB\nAQHq0aOHJCklJUVvv/22zVECAJoaElQAAFBngYGBKigoUGhoqB599FGtWrWqxnoKIQIA6sPH7gAA\nAEDTs2fPHvn5+WnUqFFq3769nn/+ee3atUs7d+5U9+7dlZmZqbi4OLvDBAA0MSSoAACgzoqKivTQ\nQw+pRYsWatWqlebMmaMffvhBSUlJKisrU1RUlO688067wwQANDHMgwoAAAAAaHDMgwoAANzO6XRq\n8OARGjx4hJxOp93hAACaMFpQAQBAvTmdTiUmpqi0dJokyeGYrJUrFyohIcHmyAAAdqtPCyoJKgAA\nqLfBg0coO3uYpJTKJQsVH/+asrJW2BkWAMAD0MUXAAAAANBkUcUXAADUW1raeOXlpai0tOK9wzFZ\naWkL7Q0KANBk0cUXAACcF6fTqYyMf0iqSFgZfwoAkBiDCgAAAADwEIxBBQAAAAA0WSSoAAAAAACP\nQIIKAAAAAPAIJKgAAAAAAI9AggoAAAAA8AgkqAAAAAAAj0CCCgAAAADwCCSoAAAAAACPQIIKAAAA\nAPAIJKgAAAAAAI9AggoAAAAA8AgkqAAAAAAAj0CCCgAAAADwCCSoAAAAAACPQIIKAAAAAPAIJKgA\nAAAAAI9AggoAAAAA8AgkqAAAAAAAj0CCCgAAAADwCCSoAAAAAACPQIIKAAAAAPAIJKgAAAAAAI9A\nggoAAAAA8AgkqAAAAAAAj0CCCgAAAADwCCSoAAAAAACPQIIKAAAAAPAIJKgAAAAAAI9AggoAAAAA\n8AgkqAAAAAAAj0CCCgAAAADwCCSoAAAAAACPQIIKAAAAAPAIJKgAAAAAAI9AggoAAAAA8AgkqAAA\nAAAAj0CCCgAAAADwCCSoAAAAAACPQIIKAAAAAPAIJKgAAAAAAI9AggoAAAAA8AgkqAAAAAAAj0CC\nCgAAAADwCCSoAAAAAACPQIIKAAAAAPAIJKgAAAAAAI9AggoAAAAA8Aj1TlCNMUnGmI+NMSeMMX2r\nLe9qjCk1xhRU/pndMKECAAAAAJqz82lBLZKUKOntWtbtsCwrovLPXefxHXCT3Nxcu0NAJc6F5+Bc\neA7OhefgXHgWzofn4Fx4Ds5F01bvBNWyrG2WZX3SkMHAPlzInoNz4Tk4F56Dc+E5OBeehfPhOTgX\nnoNz0bQ11hjUgMruvbnGmJhG+g4AAAAAQDPic7aVxphsSZfWsuphy7JWn2G3ryR1sSxrf+XY1FeN\nMSGWZR06z1gBAAAAAM2YsSzr/D7AmLckpVmWtaku640x5/fFAAAAAACPZlmWqcv2Z21BrYOqLzXG\ndJC037KsE8aYbpICJX126g51DRQAAAAA0LydzzQzicaYzyVdJekNY8yaylXXSPrIGFMg6d+Sfm9Z\n1g/nHyoAAAAAoDk77y6+AAAAAAA0hMaq4ntGxpjpxpitxpiPjDH/Mca0r1ze1RhTWln9t8AYM9vd\nsXmbM52LynV/MMZ8aozZZowZbGec3sAYk2SM+dgYc6KyuNjJ5VwXbnamc1G5juvCRsaYx40xX1S7\nHobYHZO3McYMqfz7/6kxZrLd8XgzY4zLGFNYeS18YHc83sQYM98Ys9cYU1Rt2S+NMdnGmE+MMVnG\nmIvsjNGbnOF88HthA2NMF2PMW5X3UVuMMfdVLq/T9eH2BFVSlqQQy7L6SPpE0h+qrdthWVZE5Z+7\nbIjN29R6LowxwZJGSgqWNETSbGOMHX9XvEmRpERJb9eyjuvCvWo9F1wXHsGS9Gy16+FNuwPyJsaY\nlpL+roq//8GSbjXG9LI3Kq9mSYqrvBai7A7GyyxQxXVQ3RRJ2ZZl9ZSUU/ke7lHb+eD3wh7HJU20\nLCtEFcNA7678najT9eH2myvLsrItyyqvfPu+pMvcHQMqnOVcDJe0xLKs45ZluSTtkMSPXyOyLGub\nZVmf2B0HznouuC48AwX27BOligdmLsuyjkv6lyquC9iH68EGlmW9I2n/KYuHSVpY+XqhpN+4NSgv\ndobzIXF9uJ1lWV9blrW58vVhSVsldVYdrw+7n/6PkfR/1d4HVDbD5xpjYuwKyktVPxedJH1Rbd0X\nqvjLBXtwXXgGrgvPcG/lsIR5dKFzu86SPq/2nmvAXpak/xpjNhpj7rA7GOgSy7L2Vr7eK+kSO4OB\nJH4vbGWM6SopQhWNYHW6PhpqmplTA8qWdGktqx62LGt15TaPSDpmWdbiynVfSepiWdb+ynFfrxpj\nQizLOtQYMXqLep6L2lBN6zydy7moBddFI6jnuagN10UDO8u5eUTSHElPVr7/k6QMSWPdFBr4++5p\nBlqWtccYc7GkbGPMtsqWJNjMsizLGMP1Yi9+L2xkjPmFpBWS7rcs65AxPzVmn8v10SgJqmVZ8Wdb\nb4wZLenXkq6tts8xSccqX28yxuxUxRyqmxojRm9Rn3Mh6UtJXaq9v6xyGc7Dz52LM+zDddEI6nMu\nxHXhFud6bowx/5RUl4cJOH+nXgNdVLNXAdzIsqw9lf/91hizUhVdsElQ7bPXGHOpZVlfG2M6SvrG\n7oC8mWVZVf//+b1wL2OMryqS00zLsl6tXFyn68OOKr5DJD0kabhlWUeqLe9QWYBBxphuqrgJ/8zd\n8XmTM50LSa9JusUY08oYE6CKc0GFQPepeszEdWG76uNXuC5sVvmjdlKiKgpawX02SgqsrC7eShVF\nw16zOSavZIy50BjTtvJ1G0mDxfVgt9ckpVS+TpH06lm2RSPj98IepqKpdJ6kYsuy/lZtVZ2uD7fP\ng2qM+VRSK0n7Khe9Z1nWXcaYEZKeUEX1p3JJf7Qs6w23BudlznQuKtc9rIpxqWWqaJ532hOldzDG\nJEqaJamDpAOSCizLup7rwv3OdC4q13Fd2MgY87KkcFV0Nf3/Jf2+2pgWuIEx5npJf5PUUtI8y7L+\nbHNIXqnyIdnKyrc+kl7hXLiPMWaJpGtU8TuxV9IfJa2StEzS/yfJJelmy7J+sCtGb1LL+ZgqKU78\nXrhdZa2UtyUV6qdhIX9QxQP9c74+3J6gAgAAAABQG7ur+AIAAAAAIIkEFQAAAADgIUhQAQAAAAAe\ngQQVAAAAAOARSFABAAAAAB6BBBUAAAAA4BFIUAEAAAAAHoEEFQAAAADgEf4fL2FBsQOAIcMAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x117b48890>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fig, (ax0, ax1) = plt.subplots(2,1,figsize=(16,18))\n",
    "fig, ax0 = plt.subplots()\n",
    "for i, label in enumerate(vocab[:num_points]):\n",
    "    x, y = nn_2d_embeddings[i,:]\n",
    "    ax0.scatter(x, y)\n",
    "    ax0.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "                   ha='right', va='bottom')\n",
    "    \n",
    "# for i, label in enumerate(vocab[:num_points]):\n",
    "#     x, y = levy_2d_embeddings[i,:]\n",
    "#     ax1.scatter(x, y)\n",
    "#     ax1.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n",
    "#                    ha='right', va='bottom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2], dtype=int32)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array([1.2, 2.4])\n",
    "a.astype(np.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}