3_train_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from six.moves import cPickle as pickle\n",
    "from scipy import ndimage\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pickle_file = 'images_tensorflow.pickle' \n",
    "try:\n",
    "    with open(pickle_file, 'rb') as f:\n",
    "        digits = pickle.load(f) # unpickle\n",
    "        train_X = digits['train']['feature'].reshape(-1, 32, 32, 1)\n",
    "        test_X = digits['test']['feature'].reshape(-1, 32, 32, 1)\n",
    "        valid_X = digits['valid']['feature'].reshape(-1, 32, 32, 1)\n",
    "        extra_X = digits['extra']['feature'].reshape(-1, 32, 32, 1)\n",
    "        train_Y = digits['train']['label']\n",
    "        test_Y = digits['test']['label']\n",
    "        valid_Y = digits['valid']['label']\n",
    "        extra_Y = digits['extra']['label']\n",
    "except Exception as e:\n",
    "    print('Unable to process data from', pickle_file, ':', e)\n",
    "    raise\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the final LeNet5 model with regularization [Deep Learning Assignment 4](https://github.com/yinghsienwu/DeepLearning/blob/master/4_convolutions.ipynb)\n",
    "----\n",
    "batch_size = 128, \n",
    "image_size = 32, \n",
    "num_hidden = 84, \n",
    "num_labels = 11 (0-9, blank)\n",
    "\n",
    "The model includes 7 layers:\n",
    "1. C1: convolutional, C1_depth = 6\n",
    "    - weights = batch_size x 32 x 32 x C1_depth\n",
    "    - patch = 5 x 5 x 1 x C1_depth (patch_size = 5, num_channel = 1)\n",
    "- S2: sub-sampling, stride = 2 [1, 2, 2, 1] \n",
    "    - batch_size x 16 x 16 x C1_depth\n",
    "- C3: convolutional, C3_depth = 16  [1, 2, 2, 1]\n",
    "    - weights = batch_size x 10 x 10 x C3_depth\n",
    "    - patch = 5 x 5 x C1_depth x C3_depth\n",
    "- S4: sub-sampling, stride=2 [1, 2, 2, 1]\n",
    "    - batch_size x 5 x 5 x C3_depth\n",
    "- C5: convolutional, C5_depth = 120\n",
    "    - weights = batch_size x 1 x 1 x C5_depth\n",
    "    - patch = 5 x 5 x C3_depth x C5_depth\n",
    "- F6: fully-connected\n",
    "    - C5_conv_dim = ((((image_size+1)//2+1)//2+1)//2) = 4\n",
    "    - weights = (C5_conv_dim x C5_conv_dim x C5_depth) x num_hidden\n",
    "- O7: output\n",
    "    - weights = num_hidden x num_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def accuracy(predictions, labels):\n",
    "    #return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))/predictions.shape[0])\n",
    "    return (100.0 * np.sum(np.argmax(predictions, 2).T ==labels)/predictions.shape[1]/predictions.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('shape: ', [64, 1, 1, 64])\n",
      "('hidden shape: ', [64, 64])\n",
      "('shape: ', [2000, 1, 1, 64])\n",
      "('hidden shape: ', [2000, 64])\n",
      "('shape: ', [13068, 1, 1, 64])\n",
      "('hidden shape: ', [13068, 64])\n"
     ]
    }
   ],
   "source": [
    "batch_size = 64\n",
    "patch_size = 5\n",
    "image_size = 32\n",
    "\n",
    "num_labels = 11\n",
    "num_channels = 1\n",
    "c1_depth = 16\n",
    "c3_depth = 32\n",
    "c5_depth = 64\n",
    "num_hidden = 64\n",
    "\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "    # Input data\n",
    "    tf_train_dataset = tf.placeholder(tf.float32, \n",
    "                                      shape=(batch_size, image_size, image_size, num_channels))\n",
    "    tf_train_labels = tf.placeholder(tf.int32, shape=(batch_size, 6)) #\n",
    "    tf_valid_dataset = tf.constant(valid_X)\n",
    "    tf_test_dataset = tf.constant(test_X)\n",
    "    \n",
    "    # Variables\n",
    "    c1_weights = tf.get_variable('c1w', shape=[patch_size, patch_size, num_channels, c1_depth], \n",
    "                                initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "    c1_biases = tf.Variable(tf.constant(1.0, shape=[c1_depth]), name='c1b')\n",
    "    \n",
    "    c3_weights = tf.get_variable('c3w', shape=[patch_size, patch_size, c1_depth, c3_depth],\n",
    "                             initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "    c3_biases = tf.Variable(tf.constant(1.0, shape=[c3_depth]), name='c3b')\n",
    "    \n",
    "    c5_weights = tf.get_variable('c5w', shape=[patch_size, patch_size, c3_depth, num_hidden], \n",
    "                                 initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "    c5_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]), name='c5b')\n",
    "    '''\n",
    "    c5_conv_dim = (((((image_size+1)//2)+1)//2)+1)//2\n",
    "    print(c5_conv_dim)\n",
    "    f6_weights = tf.get_variable('f6w',shape=[c5_conv_dim * c5_conv_dim * c5_depth, num_hidden],\n",
    "                                initializer=tf.contrib.layers.xavier_initializer())\n",
    "    f6_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]), name='f6b')\n",
    "    '''\n",
    "    # output layer: sequence of 5 digits\n",
    "    d1_weights = tf.get_variable('d1w', shape=[num_hidden, num_labels], initializer=tf.contrib.layers.xavier_initializer())\n",
    "    d1_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='d1b')\n",
    "    d2_weights = tf.get_variable('d2w', shape=[num_hidden, num_labels], initializer=tf.contrib.layers.xavier_initializer())\n",
    "    d2_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='d2b')\n",
    "    d3_weights = tf.get_variable('d3w', shape=[num_hidden, num_labels], initializer=tf.contrib.layers.xavier_initializer())\n",
    "    d3_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='d3b')\n",
    "    d4_weights = tf.get_variable('d4w', shape=[num_hidden, num_labels], initializer=tf.contrib.layers.xavier_initializer())\n",
    "    d4_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='d4b')\n",
    "    d5_weights = tf.get_variable('d5w', shape=[num_hidden, num_labels], initializer=tf.contrib.layers.xavier_initializer())\n",
    "    d5_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='d5b')\n",
    "    \n",
    "    # Model\n",
    "    def model(data, keep_prob=1.0):\n",
    "        conv = tf.nn.conv2d(data, c1_weights, [1, 1, 1, 1], padding='VALID')\n",
    "        hidden = tf.nn.relu(conv + c1_biases)\n",
    "        lrn = tf.nn.local_response_normalization(hidden)\n",
    "        maxpool = tf.nn.max_pool(lrn, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "        \n",
    "        conv = tf.nn.conv2d(maxpool, c3_weights, [1, 1, 1, 1], padding='VALID')\n",
    "        hidden = tf.nn.relu(conv + c3_biases)\n",
    "        lrn = tf.nn.local_response_normalization(hidden)\n",
    "        maxpool = tf.nn.max_pool(lrn, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "        \n",
    "        conv = tf.nn.conv2d(maxpool, c5_weights, [1, 1, 1, 1], padding='VALID')\n",
    "        hidden = tf.nn.relu(conv + c5_biases)\n",
    "        \n",
    "        drop = tf.nn.dropout(hidden, keep_prob)\n",
    "        \n",
    "        shape = drop.get_shape().as_list()\n",
    "        print('shape: ',shape)\n",
    "        hidden = tf.reshape(drop, [shape[0], shape[1]*shape[2]*shape[3]])\n",
    "        #hidden = tf.nn.relu(tf.matmul(reshape, f6_weights) + f6_biases)\n",
    "        print('hidden shape: ',hidden.get_shape().as_list())\n",
    "        l1 = tf.matmul(hidden, d1_weights) + d1_biases\n",
    "        l2 = tf.matmul(hidden, d2_weights) + d2_biases\n",
    "        l3 = tf.matmul(hidden, d3_weights) + d3_biases\n",
    "        l4 = tf.matmul(hidden, d4_weights) + d4_biases\n",
    "        l5 = tf.matmul(hidden, d5_weights) + d5_biases\n",
    "        return [l1 ,l2, l3, l4, l5]\n",
    "\n",
    "    # Training computation\n",
    "    [l1 ,l2, l3, l4, l5] = model(tf_train_dataset) # keep_prob=1.0\n",
    "    # L-2 regularization\n",
    "    #beta = 0 #1e-3\n",
    "    #R = tf.nn.l2_loss(c1_weights) + tf.nn.l2_loss(c3_weights) + tf.nn.l2_loss(c5_weights)\n",
    "    # tf.nn.sparse_softmax_cross_entropy_with_logits, no need 1-hot encoding\n",
    "    loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(l1, tf_train_labels[:,1]) +\\\n",
    "    tf.nn.sparse_softmax_cross_entropy_with_logits(l2, tf_train_labels[:,2]) + \\\n",
    "    tf.nn.sparse_softmax_cross_entropy_with_logits(l3, tf_train_labels[:,3]) + \\\n",
    "    tf.nn.sparse_softmax_cross_entropy_with_logits(l4, tf_train_labels[:,4]) + \\\n",
    "    tf.nn.sparse_softmax_cross_entropy_with_logits(l5, tf_train_labels[:,5]))\n",
    "    \n",
    "    # Optimizer with exponential decay learning rate\n",
    "    global_step = tf.Variable(0)\n",
    "    learning_rate = tf.train.exponential_decay(0.05, global_step, 100, 0.96)\n",
    "    optimizer = tf.train.AdagradOptimizer(learning_rate).minimize(loss, global_step=global_step)\n",
    "    \n",
    "    # Predictions for the training, validation, and test data\n",
    "    train_prediction = tf.pack([tf.nn.softmax(l1), tf.nn.softmax(l2), tf.nn.softmax(l3),\n",
    "                                tf.nn.softmax(l4), tf.nn.softmax(l5)])\n",
    "    [vl1, vl2, vl3, vl4, vl5] = model(tf_valid_dataset)\n",
    "    valid_prediction = tf.pack([tf.nn.softmax(vl1), tf.nn.softmax(vl2), tf.nn.softmax(vl3),\n",
    "                                tf.nn.softmax(vl4), tf.nn.softmax(vl5)])\n",
    "    [tl1, tl2, tl3, tl4, tl5] = model(tf_test_dataset)\n",
    "    test_prediction = tf.pack([tf.nn.softmax(tl1), tf.nn.softmax(tl2), tf.nn.softmax(tl3),\n",
    "                                tf.nn.softmax(tl4), tf.nn.softmax(tl5)])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 20.761406\n",
      "Minibatch accuracy: 22.5%\n",
      "Validation accuracy: 70.6%\n",
      "Minibatch loss at step 2000: 3.722440\n",
      "Minibatch accuracy: 76.9%\n",
      "Validation accuracy: 71.6%\n",
      "Minibatch loss at step 4000: 3.419424\n",
      "Minibatch accuracy: 77.8%\n",
      "Validation accuracy: 71.6%\n",
      "Minibatch loss at step 6000: 3.312357\n",
      "Minibatch accuracy: 77.5%\n",
      "Validation accuracy: 71.5%\n",
      "Minibatch loss at step 8000: 3.126363\n",
      "Minibatch accuracy: 79.7%\n",
      "Validation accuracy: 71.4%\n",
      "Minibatch loss at step 10000: 3.133566\n",
      "Minibatch accuracy: 81.9%\n",
      "Validation accuracy: 71.3%\n",
      "Minibatch loss at step 12000: 3.787938\n",
      "Minibatch accuracy: 75.6%\n",
      "Validation accuracy: 71.4%\n",
      "Minibatch loss at step 14000: 3.424648\n",
      "Minibatch accuracy: 78.8%\n",
      "Validation accuracy: 71.3%\n",
      "Minibatch loss at step 16000: 2.987230\n",
      "Minibatch accuracy: 80.0%\n",
      "Validation accuracy: 71.4%\n",
      "Minibatch loss at step 18000: 3.039293\n",
      "Minibatch accuracy: 80.6%\n",
      "Validation accuracy: 71.4%\n",
      "Test accuracy: 82.9%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 20000\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.initialize_all_variables().run()\n",
    "\n",
    "  print('Initialized')\n",
    "  for step in range(num_steps):\n",
    "    offset = (step * batch_size) % (train_Y.shape[0] - batch_size)\n",
    "    batch_data = train_X[offset:(offset + batch_size), :, :, :]\n",
    "    batch_labels = train_Y[offset:(offset + batch_size),:]\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run(\n",
    "      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    \n",
    "    if (step % 2000 == 0): \n",
    "      print('Minibatch loss at step %d: %f' % (step, l))\n",
    "      print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels[:,1:6]))\n",
    "      print('Validation accuracy: %.1f%%' % accuracy(valid_prediction.eval(), valid_Y[:,1:6]))\n",
    "  print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_Y[:,1:6]))\n",
    "  #save_path = saver.save(session, \"SVHN_MODEL.ckpt\")\n",
    "  #print(\"Model saved in file: %s\" % save_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}