Merge pull request #8 from rsepassi/master

v1.0.4

README.md

@@ -11,7 +11,9 @@ welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg)](CO
 [T2T](https://github.com/tensorflow/tensor2tensor) is a modular and extensible
 library and binaries for supervised learning with TensorFlow and with support
 for sequence tasks. It is actively used and maintained by researchers and
-engineers within the Google Brain team.
+engineers within the Google Brain team. You can read more about Tensor2Tensor in
+the recent [Google Research Blog post introducing
+it](https://research.googleblog.com/2017/06/accelerating-deep-learning-research.html).
 
 We're eager to collaborate with you on extending T2T, so please feel
 free to [open an issue on
@@ -50,6 +52,7 @@ mkdir -p $DATA_DIR $TMP_DIR $TRAIN_DIR
 t2t-datagen \
   --data_dir=$DATA_DIR \
   --tmp_dir=$TMP_DIR \
+  --num_shards=100 \
   --problem=$PROBLEM
 
 mv $TMP_DIR/tokens.vocab.32768 $DATA_DIR

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name='tensor2tensor',
-    version='1.0.3',
+    version='1.0.4',
     description='Tensor2Tensor',
     author='Google Inc.',
     author_email='[email protected]',
@@ -26,4 +26,4 @@
         'License :: OSI Approved :: Apache Software License',
         'Topic :: Scientific/Engineering :: Artificial Intelligence',
     ],
-    keywords='tensorflow',)
+    keywords='tensorflow machine learning',)

tensor2tensor/bin/t2t-datagen

@@ -51,7 +51,7 @@ flags.DEFINE_string("tmp_dir", "/tmp/t2t_datagen",
                     "Temporary storage directory.")
 flags.DEFINE_string("problem", "",
                     "The name of the problem to generate data for.")
-flags.DEFINE_integer("num_shards", 1, "How many shards to use.")
+flags.DEFINE_integer("num_shards", 10, "How many shards to use.")
 flags.DEFINE_integer("max_cases", 0,
                      "Maximum number of cases to generate (unbounded if 0).")
 flags.DEFINE_integer("random_seed", 429459, "Random seed to use.")

tensor2tensor/data_generators/algorithmic.py

@@ -43,7 +43,7 @@ def identity_generator(nbr_symbols, max_length, nbr_cases):
   for _ in xrange(nbr_cases):
     l = np.random.randint(max_length) + 1
     inputs = [np.random.randint(nbr_symbols) + 2 for _ in xrange(l)]
-    yield {"inputs": inputs, "targets": inputs}
+    yield {"inputs": inputs, "targets": inputs + [1]}  # [1] for EOS
 
 
 def shift_generator(nbr_symbols, shift, max_length, nbr_cases):
@@ -66,7 +66,8 @@ def shift_generator(nbr_symbols, shift, max_length, nbr_cases):
   for _ in xrange(nbr_cases):
     l = np.random.randint(max_length) + 1
     inputs = [np.random.randint(nbr_symbols - shift) + 2 for _ in xrange(l)]
-    yield {"inputs": inputs, "targets": [i + shift for i in inputs]}
+    yield {"inputs": inputs,
+           "targets": [i + shift for i in inputs] + [1]}  # [1] for EOS
 
 
 def reverse_generator(nbr_symbols, max_length, nbr_cases):
@@ -88,7 +89,8 @@ def reverse_generator(nbr_symbols, max_length, nbr_cases):
   for _ in xrange(nbr_cases):
     l = np.random.randint(max_length) + 1
     inputs = [np.random.randint(nbr_symbols) + 2 for _ in xrange(l)]
-    yield {"inputs": inputs, "targets": list(reversed(inputs))}
+    yield {"inputs": inputs,
+           "targets": list(reversed(inputs)) + [1]}  # [1] for EOS
 
 
 def lower_endian_to_number(l, base):
@@ -141,7 +143,7 @@ def addition_generator(base, max_length, nbr_cases):
     # We shift digits by 1 on input and output to leave 0 for padding.
     inputs = [i + 2 for i in n1] + [base + 2] + [i + 2 for i in n2]
     targets = [i + 2 for i in number_to_lower_endian(result, base)]
-    yield {"inputs": inputs, "targets": targets}
+    yield {"inputs": inputs, "targets": targets + [1]}  # [1] for EOS
 
 
 def multiplication_generator(base, max_length, nbr_cases):
@@ -175,4 +177,4 @@ def multiplication_generator(base, max_length, nbr_cases):
     # We shift digits by 1 on input and output to leave 0 for padding.
     inputs = [i + 2 for i in n1] + [base + 2] + [i + 2 for i in n2]
     targets = [i + 2 for i in number_to_lower_endian(result, base)]
-    yield {"inputs": inputs, "targets": targets}
+    yield {"inputs": inputs, "targets": targets + [1]}  # [1] for EOS

tensor2tensor/data_generators/algorithmic_test.py

@@ -31,14 +31,14 @@ def testIdentityGenerator(self):
     counter = 0
     for d in algorithmic.identity_generator(3, 8, 10):
       counter += 1
-      self.assertEqual(d["inputs"], d["targets"])
+      self.assertEqual(d["inputs"] + [1], d["targets"])
     self.assertEqual(counter, 10)
 
   def testReverseGenerator(self):
     counter = 0
     for d in algorithmic.reverse_generator(3, 8, 10):
       counter += 1
-      self.assertEqual(list(reversed(d["inputs"])), d["targets"])
+      self.assertEqual(list(reversed(d["inputs"])) + [1], d["targets"])
     self.assertEqual(counter, 10)
 
   def testLowerEndianToNumber(self):
@@ -63,19 +63,19 @@ def testAdditionGenerator(self):
     counter = 0
     for d in algorithmic.addition_generator(4, 8, 10):
       counter += 1
-      self.assertEqual(d["inputs"].count(5), 1)
+      self.assertEqual(d["inputs"].count(6), 1)
       self.assertEqual(d["inputs"].count(0), 0)
-      self.assertEqual(d["targets"].count(5), 0)
+      self.assertEqual(d["targets"].count(6), 0)
       self.assertEqual(d["targets"].count(0), 0)
     self.assertEqual(counter, 10)
 
   def testMultiplicationGenerator(self):
     counter = 0
     for d in algorithmic.multiplication_generator(4, 8, 10):
       counter += 1
-      self.assertEqual(d["inputs"].count(5), 1)
+      self.assertEqual(d["inputs"].count(6), 1)
       self.assertEqual(d["inputs"].count(0), 0)
-      self.assertEqual(d["targets"].count(5), 0)
+      self.assertEqual(d["targets"].count(6), 0)
       self.assertEqual(d["targets"].count(0), 0)
     self.assertEqual(counter, 10)