JUnit5: Implement @Seed for classes

datagen/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>qala-datagen-parent</artifactId>
         <groupId>io.qala.datagen</groupId>
-        <version>1.14.0</version>
+        <version>2.0.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

examples/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>qala-datagen-parent</artifactId>
         <groupId>io.qala.datagen</groupId>
-        <version>1.14.0</version>
+        <version>2.0.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

java8types/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>qala-datagen-parent</artifactId>
         <groupId>io.qala.datagen</groupId>
-        <version>1.14.0</version>
+        <version>2.0.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

junit5/README.md

@@ -48,14 +48,84 @@ Though if you need to run a test only once and you want to use randomization - i
 [Datagen API](./../README.md) directly. More examples are available in 
 [the test](src/test/java/io/qala/datagen/junit5/Junit5ParameterizedTest.java).
 
+# Seeds
+
+Seed is a number that determines which random values are generated. By default it's initialized with `System.nanoTime()`
+but it's possible to hardcode it, in that case every time the test is run the same random data is going to be generated:
+
+```java
+@Test @Seed(123)
+void explicitSeed_generatesSameDataEveryTime() {
+    assertEquals("56847945", numeric(integer(1, 10)));
+    assertEquals("0o2V9KpUJc6", alphanumeric(integer(1, 20)));
+    assertEquals("lfBi", english(1, 10));
+    assertEquals(1876573356364443993L, Long());
+    assertEquals(-8.9316016195567002E18, Double());
+    assertEquals(false, bool());
+}
+```
+
+This will work for parameterized tests as well. It's possible to set the `@Seed` per class which would make all the
+test methods generate the same values over and over again.
+
+Usually you don't need to set the seed, but if your test fails it's nice if you could reproduce it exactly again -
+that's the primary use case for setting seeds manually. If you add this extension to your test classes it will put the
+method and class seeds into logs (use SLF4J implementations) if a test fails:
+
+```java
+@ExtendWith(DatagenSeedExtension.class)
+class Junit5ParameterizedTest {}
+```
+
+According to [JUnit5 docs](http://junit.org/junit5/docs/current/user-guide/#extensions-registration) you can pass system
+argument to enable auto-registration instead of marking each class with `@ExtendWith`:
+
+```
+-Djunit.extensions.autodetection.enabled=true
+```
+
+The output may look something like this:
+
+```
+Random Seeds:  testMethod[162024700321388] NestedTestClass[162024700321388] EnclosingTestClass[286157404280696]
+```
+
+
+## Seed Caveats
+
+1. Right now JUnit5 [doesn't have a deterministic order of tests](https://github.com/junit-team/junit5/issues/111#issuecomment-307644332).
+This means that if you run your tests twice the order can change. If you put `@Seed` on your class and that
+actually happens - it will generate different data in some cases.
+This is especially possible if your tests start with the same name and then are followed by underscores. So if you
+use `@Seed` you may need to run the test class multiple times before the order repeats. Or just put the annotation on
+methods instead.
+
+2. If you use `@MethodSource` the `@Seed` is applied only after the data came from the data methods. This is due to
+[current restrictions of JUnit5](https://github.com/junit-team/junit5/issues/883) which doesn't have any callbacks
+to impact `@MethodSource`. So this seed will not work:
+
+```
+@Seed(1234)
+@ParameterizedTest
+@MethodSource("numericsMethod")
+void test(String value) {
+    assertFalse(value.contains(english(1)));
+}
+private static Stream<? extends Arguments> numericsMethod() {
+    return Stream.of(numeric(10)).map(Arguments::of);
+}
+```
+
+# Add to your project
+
 This integration is not stable yet because JUnit5 itself is not stable. But if you're not afraid of the fact that the 
 API may change in the future, you can give it a try. In order for this to work you need the latest snapshot of JUnit5:
 ```xml
 <dependencies>
     <dependency>
         <groupId>io.qala.datagen</groupId>
         <artifactId>qala-datagen-junit5</artifactId>
-        <version>1.14.0</version>
+        <version>2.0.0</version>
         <scope>test</scope>
     </dependency>
 

junit5/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>qala-datagen-parent</artifactId>
         <groupId>io.qala.datagen</groupId>
-        <version>1.14.0</version>
+        <version>2.0.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -16,6 +16,14 @@
             <groupId>io.qala.datagen</groupId>
             <artifactId>qala-datagen</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.junit.platform</groupId>
             <artifactId>junit-platform-commons</artifactId>

junit5/src/main/java/io/qala/datagen/junit5/DatagenUtils.java

@@ -4,57 +4,82 @@
 import io.qala.datagen.Seed;
 import org.junit.jupiter.api.extension.ContainerExtensionContext;
 import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ExtensionContext.Namespace;
+import org.junit.jupiter.api.extension.ExtensionContext.Store;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.lang.reflect.Method;
+import java.util.LinkedHashMap;
+import java.util.Map;
 import java.util.Optional;
 
 public class DatagenUtils {
+    private static final Logger LOG = LoggerFactory.getLogger(DatagenUtils.class);
+
     static boolean passCaseNameToTestMethod(ContainerExtensionContext context) {
         Optional<Method> testMethod = context.getTestMethod();
         return testMethod.filter(method -> method.getParameterCount() == 2).isPresent();
     }
 
     public static void setCurrentSeedIfNotSetYet(ExtensionContext context) {
-        ExtensionContext.Store classStore = getClassStore(context);
-        if(classStore != null) {
-            Long currentClassSeed = classStore.get("current_class_seed", Long.class);
-            if (currentClassSeed == null && context.getTestClass().isPresent()) {
-                Seed classAnnotation = context.getTestClass().get().getAnnotation(Seed.class);
-                if (classAnnotation != null) DatagenRandom.overrideSeed(classAnnotation.value());
-                classStore.put("current_class_seed", DatagenRandom.getCurrentSeed());
-            }
-        }
+        if(getCurrentLevelSeedFromStore(context) != null) return;
 
-        ExtensionContext.Store methodStore = getMethodStore(context);
-        if(methodStore != null) {
-            Long currentMethodSeed = methodStore.get("current_method_seed", Long.class);
-            if (currentMethodSeed == null && context.getTestMethod().isPresent()) {
-                Seed methodAnnotation = context.getTestMethod().get().getAnnotation(Seed.class);
-                if (methodAnnotation != null) DatagenRandom.overrideSeed(methodAnnotation.value());
-                methodStore.put("current_method_seed", DatagenRandom.getCurrentSeed());
-            }
+        Optional<Method> testMethod = context.getTestMethod();
+        Optional<Class<?>> testClass = context.getTestClass();
+        if(testMethod.isPresent()) {
+            Seed annotation = testMethod.get().getAnnotation(Seed.class);
+            if (annotation != null) DatagenRandom.overrideSeed(annotation.value());
+        } else if(testClass.isPresent()) {
+            Seed annotation = testClass.get().getAnnotation(Seed.class);
+            if (annotation != null) DatagenRandom.overrideSeed(annotation.value());
         }
+        putSeedToStoreIfAbsent(context, DatagenRandom.getCurrentSeed());
     }
 
     public static void logCurrentSeeds(ExtensionContext context) {
-        ExtensionContext.Store methodStore = getMethodStore(context);
-        ExtensionContext.Store classStore = getClassStore(context);
-        String logLine = "";
-        if (methodStore != null) logLine = " Method Seed: [" + methodStore.get("current_method_seed", Long.class) + "]";
-        if (classStore != null) logLine += " Class Seed: [" + classStore.get("current_class_seed", Long.class) + "]";
-        if (!logLine.isEmpty()) logLine = "Datagen" + logLine + " for " + context.getUniqueId();
-        System.out.println(logLine);
+        LinkedHashMap<String, Long> seedStrings = new LinkedHashMap<>();
+        while(context != null) {
+            Long seed = getCurrentLevelSeedFromStore(context);
+            Optional<Method> testMethod = context.getTestMethod();
+            Optional<Class<?>> testClass = context.getTestClass();
+            if(testMethod.isPresent()) seedStrings.putIfAbsent(testMethod.get().getName(), seed);
+            else                       testClass.ifPresent((c)->seedStrings.put(c.getSimpleName(), seed));
+
+            context = context.getParent().isPresent() ? context.getParent().get() : null;
+        }
+        StringBuilder logLine = new StringBuilder();
+        for(Map.Entry<String, Long> seed: seedStrings.entrySet()) {
+            logLine.append(" ").append(seed.getKey()).append("[").append(seed.getValue()).append("]");
+        }
+        if(logLine.length() != 0) LOG.info("Random Seeds: {}", logLine);
+    }
+
+    private static void putSeedToStoreIfAbsent(ExtensionContext context, Long seed) {
+        Store store = getStore(context);
+        if(store != null) store.put("seed", seed);
+    }
+    private static Long getCurrentLevelSeedFromStore(ExtensionContext context) {
+        Store store = getStore(context);
+        Object seed = null;
+        if(store != null) seed = store.get("seed");
+        return seed != null ? (Long) seed : null;
+    }
+    private static Store getStore(ExtensionContext context) {
+        Store store = getMethodStore(context);
+        if(store == null) store = getClassStore(context);
+        return store;
     }
 
-    private static ExtensionContext.Store getMethodStore(ExtensionContext context) {
+    private static Store getMethodStore(ExtensionContext context) {
         Optional<Method> m = context.getTestMethod();
-        if (m.isPresent()) return context.getStore(ExtensionContext.Namespace.create(DatagenUtils.class, m.get()));
+        if (m.isPresent()) return context.getStore(Namespace.create(DatagenUtils.class, m.get()));
         return null;
     }
 
-    private static ExtensionContext.Store getClassStore(ExtensionContext context) {
+    private static Store getClassStore(ExtensionContext context) {
         Optional<Class<?>> c = context.getTestClass();
-        if (c.isPresent()) return context.getStore(ExtensionContext.Namespace.create(DatagenUtils.class, c.get()));
+        if (c.isPresent()) return context.getStore(Namespace.create(DatagenUtils.class, c.get()));
         return null;
     }
 }

junit5/src/main/java/io/qala/datagen/junit5/seed/DatagenSeedExtension.java

@@ -3,6 +3,14 @@
 import io.qala.datagen.junit5.DatagenUtils;
 import org.junit.jupiter.api.extension.*;
 
+/**
+ * Logs the seeds of the failed methods so that it's possible to re-run tests with the same data generated.
+ * Unfortunately right now
+ * <a href="https://github.com/junit-team/junit5/issues/618">JUnit5 doesn't have means to know which test failed</a>
+ * and which passed, so currently the seed is printed for each test method that threw exception.
+ *
+ * @see io.qala.datagen.Seed
+ */
 public class DatagenSeedExtension implements BeforeTestExecutionCallback, BeforeAllCallback, TestExecutionExceptionHandler {
     @Override
     public void handleTestExecutionException(TestExtensionContext context, Throwable throwable) throws Throwable {
@@ -13,6 +21,7 @@ public void handleTestExecutionException(TestExtensionContext context, Throwable
     @Override public void beforeTestExecution(TestExtensionContext context) throws Exception {
         context.getTestMethod().ifPresent((m) -> DatagenUtils.setCurrentSeedIfNotSetYet(context));
     }
+
     @Override public void beforeAll(ContainerExtensionContext context) throws Exception {
         context.getTestClass().ifPresent((m) -> DatagenUtils.setCurrentSeedIfNotSetYet(context));
     }

junit5/src/test/java/io/qala/datagen/junit5/Junit5ParameterizedTest.java

@@ -1,17 +1,23 @@
 package io.qala.datagen.junit5;
 
+import io.qala.datagen.Seed;
 import io.qala.datagen.junit5.seed.DatagenSeedExtension;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
-import static io.qala.datagen.RandomShortApi.english;
-import static io.qala.datagen.RandomShortApi.numeric;
+import java.util.stream.Stream;
+
+import static io.qala.datagen.RandomShortApi.*;
 import static org.junit.jupiter.api.Assertions.*;
 
 @SuppressWarnings("unused"/*some tests just check that the method param doesn't break anything but don't use it*/)
-@ExtendWith(DatagenSeedExtension.class)
 @DisplayName("JUnit5 Parameterized")
+@ExtendWith(DatagenSeedExtension.class)
 class Junit5ParameterizedTest {
     @Nested class IntGenerator {
         @RandomInt(min = 1, max = 10)
@@ -361,8 +367,8 @@ void nameIsGeneric_byDefault(String value, String name) {
         @Unicode(min = 90) void generatesStringWithCorrectSymbols(String string) {
             assertNotEquals(string.getBytes().length, string.length());
         }
-        @Unicode(length = 100) void generatesStringWithCorrectSymbols_whenLengthUsed(String string) {
-            assertNotEquals(string.getBytes().length, string.length());
+        @Unicode(length = 100) void generatesStringWithCorrectSymbols_whenLengthUsed(String string) throws Exception {
+            assertNotEquals(string.getBytes("UTF-8").length, string.length(), "value: " +string);
         }
 
         @Unicode(name = "name") void setsNameAsArgumentIf2ndParameterExists(String value, String name) {
@@ -415,6 +421,30 @@ void nameIsGeneric_byDefault(String value, String name) {
         }
     }
 
+    // Such tests are impossible to set seed for as for now, JUnit5 doesn't have callbacks that are run before
+    // Argument Providers are invoked.
+    @Seed(1234)
+    @ParameterizedTest
+    @MethodSource("numericsMethod")
+    void test(String value) {
+        assertFalse(value.contains(english(1)));
+    }
+    private static Stream<? extends Arguments> numericsMethod() {
+        return Stream.of(numeric(10)).map(Arguments::of);
+    }
+
+    @Test @Seed(123)
+    void explicitSeed_generatesSameDataEveryTime() {
+        assertEquals("56847945", numeric(integer(1, 10)));
+        assertEquals("0o2V9KpUJc6", alphanumeric(integer(1, 20)));
+        assertEquals("lfBi", english(1, 10));
+        assertEquals(1876573356364443993L, Long());
+        assertEquals(-8.9316016195567002E18, Double());
+        assertEquals(false, bool());
+        assertEquals("\uF26B喕", unicode(integer(1, 5)));
+    }
+
+
     private static void assertChangedFromLastTime(Object newValue) {
         assertNotEquals(PREV_VALUE, newValue);
         PREV_VALUE = newValue;

pom.xml

@@ -7,7 +7,7 @@
     <groupId>io.qala.datagen</groupId>
     <artifactId>qala-datagen-parent</artifactId>
     <packaging>pom</packaging>
-    <version>1.14.0</version>
+    <version>2.0.0</version>
 
     <licenses>
         <license>
@@ -187,6 +187,18 @@
                 <artifactId>qala-datagen-junit5</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+                <version>1.7.13</version>
+            </dependency>
+            <dependency>
+                <groupId>ch.qos.logback</groupId>
+                <artifactId>logback-classic</artifactId>
+                <version>1.2.3</version>
+                <scope>test</scope>
+            </dependency>
+
             <dependency>
                 <groupId>org.junit.platform</groupId>
                 <artifactId>junit-platform-surefire-provider</artifactId>