Merge pull request #2280 from richardTingle/#2279-screenshot-tests

#2279 screenshot tests

.github/workflows/main.yml

@@ -56,7 +56,49 @@ on:
     types: [published]
 
 jobs:
-
+  ScreenshotTests:
+    name: Run Screenshot Tests
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up JDK 17
+      uses: actions/setup-java@v4
+      with:
+        java-version: '17'
+        distribution: 'temurin'
+    - name: Install Mesa3D
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mesa-utils libgl1-mesa-dri libgl1 libglx-mesa0 xvfb
+    - name: Set environment variables for Mesa3D
+      run: |
+        echo "LIBGL_ALWAYS_SOFTWARE=1" >> $GITHUB_ENV
+        echo "MESA_LOADER_DRIVER_OVERRIDE=llvmpipe" >> $GITHUB_ENV
+    - name: Start xvfb
+      run: |
+        sudo Xvfb :99 -ac -screen 0 1024x768x16 &
+        export DISPLAY=:99
+        echo "DISPLAY=:99" >> $GITHUB_ENV
+    - name: Verify Mesa3D Installation
+      run: |
+        glxinfo | grep "OpenGL"
+    - name: Validate the Gradle wrapper
+      uses: gradle/actions/wrapper-validation@v3
+    - name: Test with Gradle Wrapper
+      run: |
+        ./gradlew :jme3-screenshot-test:screenshotTest
+    - name: Upload Test Reports
+      uses: actions/upload-artifact@master
+      if: always()
+      with:
+        name: screenshot-test-report
+        retention-days: 30
+        path: |
+          **/build/reports/**
+          **/build/changed-images/**
+          **/build/test-results/**
   # Build the natives on android
   BuildAndroidNatives:
     name: Build natives for android

jme3-screenshot-tests/README.md

@@ -0,0 +1,50 @@
+# jme3-screenshot-tests
+
+This module contains tests that compare screenshots of the JME3 test applications to reference images. The tests are run using 
+the following command:
+
+```
+ ./gradlew :jme3-screenshot-test:screenshotTest
+```
+
+This will create a report in `jme3-screenshot-test/build/reports/ScreenshotDiffReport.html` that shows the differences between the reference images and the screenshots taken during the test run. Note that this is an ExtentReport. 
+
+This is most reliable when run on the CI server. The report can be downloaded from the artifacts section of the pipeline (once the full pipeline has completed). If you go into
+the Actions tab (on GitHub) and find your pipeline you can download the report from the Artifacts section. It will be called screenshot-test-report.
+
+## Machine variability
+
+It is important to be aware that the tests are sensitive to machine variability. Different GPUs may produce subtly different pixel outputs
+(that look identical to a human user). The tests are run on a specific machine and the reference images are generated on that machine. If the tests are run on a different machine, the images may not match the reference images and this is "fine". If you run these on your local machine compare the differences by eye in the report, don't wory about failing tests.
+
+## Parameterised tests
+
+By default, the tests use the class and method name to produce the screenshot image name. E.g. org.jmonkeyengine.screenshottests.effects.TestExplosionEffect.testExplosionEffect_f15.png is the testExplosionEffect test at frame 15. If you are using parameterised tests this won't work (as all the tests have the same function name). In this case you should specify the image name (including whatever parameterised information to make it unique). E.g.
+
+```
+    screenshotTest(
+        ....
+    ).setFramesToTakeScreenshotsOn(45)
+    .setBaseImageFileName("some_unique_name_" + theParameterGivenToTest)
+    .run();
+)
+```
+
+## Non-deterministic (and known bad) tests
+
+By default, screenshot variability will cause the pipeline to fail. If a test is non-deterministic (e.g. includes randomness) or 
+is a known accepted failure (that will be fixed "at some point" but not now) that can be non-desirable. In that case you can 
+change the behaviour of the test such that these are marked as warnings in the generated report but don't fail the test
+
+```
+    screenshotTest(
+        ....
+    ).setFramesToTakeScreenshotsOn(45)
+    .setTestType(TestType.NON_DETERMINISTIC)
+    .run();
+)
+```
+
+## Accepting new images
+
+It may be the case that a change makes an improvement to the library (or the test is entirely new) and the new image should be accepted as the new reference image. To do this, copy the new image to the `src/test/resources` directory. The new image can be found in the `build/changed-images` directory, however it is very important that the image come from the reference machine. This can be obtained from the CI server. The job runs only if there is an active pull request (to one of the mainline branches; e.g. master or 3.7). If you go into the Actions tab and find your pipeline you can download the report and changed images from the Artifacts section.

jme3-screenshot-tests/build.gradle

@@ -0,0 +1,38 @@
+plugins {
+    id 'java'
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation project(':jme3-desktop')
+    implementation project(':jme3-core')
+    implementation project(':jme3-effects')
+    implementation project(':jme3-terrain')
+    implementation project(':jme3-lwjgl3')
+    implementation project(':jme3-plugins')
+
+    implementation 'com.aventstack:extentreports:5.1.1'
+    implementation platform('org.junit:junit-bom:5.9.1')
+    implementation 'org.junit.jupiter:junit-jupiter'
+    testRuntimeOnly project(':jme3-testdata')
+}
+
+tasks.register("screenshotTest", Test) {
+    useJUnitPlatform{
+        filter{
+            includeTags 'integration'
+        }
+    }
+}
+
+
+test {
+    useJUnitPlatform{
+        filter{
+            excludeTags 'integration'
+        }
+    }
+}

jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/App.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2024 jMonkeyEngine
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of 'jMonkeyEngine' nor the names of its contributors
+ *   may be used to endorse or promote products derived from this software
+ *   without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.jmonkeyengine.screenshottests.testframework;
+
+import com.jme3.app.SimpleApplication;
+import com.jme3.app.state.AppState;
+import com.jme3.app.state.VideoRecorderAppState;
+import com.jme3.math.ColorRGBA;
+
+/**
+ * The app used for the tests. AppState(s) are used to inject the actual test code.
+ * @author Richard Tingle (aka richtea)
+ */
+public class App extends SimpleApplication {
+
+    public App(AppState... initialStates){
+        super(initialStates);
+    }
+
+    @Override
+    public void simpleInitApp(){
+        getViewPort().setBackgroundColor(ColorRGBA.Black);
+        setTimer(new VideoRecorderAppState.IsoTimer(60));
+    }
+
+}

jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ExtentReportExtension.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2024 jMonkeyEngine
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of 'jMonkeyEngine' nor the names of its contributors
+ *   may be used to endorse or promote products derived from this software
+ *   without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.jmonkeyengine.screenshottests.testframework;
+
+import com.aventstack.extentreports.ExtentReports;
+import com.aventstack.extentreports.ExtentTest;
+import com.aventstack.extentreports.reporter.ExtentSparkReporter;
+import com.aventstack.extentreports.reporter.configuration.Theme;
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.BeforeTestExecutionCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.TestWatcher;
+
+import java.util.Optional;
+
+/**
+ * This creates the Extent report and manages the test lifecycle
+ *
+ * @author Richard Tingle (aka richtea)
+ */
+public class ExtentReportExtension implements BeforeAllCallback, AfterAllCallback, TestWatcher, BeforeTestExecutionCallback{
+    private static ExtentReports extent;
+    private static final ThreadLocal<ExtentTest> test = new ThreadLocal<>();
+
+    @Override
+    public void beforeAll(ExtensionContext context) {
+        if(extent==null){
+            ExtentSparkReporter spark = new ExtentSparkReporter("build/reports/ScreenshotDiffReport.html");
+            spark.config().setTheme(Theme.STANDARD);
+            spark.config().setDocumentTitle("Screenshot Test Report");
+            spark.config().setReportName("Screenshot Test Report");
+            extent = new ExtentReports();
+            extent.attachReporter(spark);
+        }
+    }
+
+    @Override
+    public void afterAll(ExtensionContext context) {
+        /*
+        * this writes the entire report after each test class. This sucks but I don't think there is
+        * anywhere else I can hook into the lifecycle of the end of all tests to write the report.
+        */
+        extent.flush();
+    }
+
+    @Override
+    public void testSuccessful(ExtensionContext context) {
+        getCurrentTest().pass("Test passed");
+    }
+
+    @Override
+    public void testFailed(ExtensionContext context, Throwable cause) {
+        getCurrentTest().fail(cause);
+    }
+
+    @Override
+    public void testAborted(ExtensionContext context, Throwable cause) {
+        getCurrentTest().skip("Test aborted " + cause.toString());
+    }
+
+    @Override
+    public void testDisabled(ExtensionContext context, Optional<String> reason) {
+        getCurrentTest().skip("Test disabled: " + reason.orElse("No reason"));
+    }
+
+    @Override
+    public void beforeTestExecution(ExtensionContext context) {
+        String testName = context.getDisplayName();
+        test.set(extent.createTest(testName));
+    }
+
+    public static ExtentTest getCurrentTest() {
+        return test.get();
+    }
+}

jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/PixelSamenessDegree.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2024 jMonkeyEngine
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of 'jMonkeyEngine' nor the names of its contributors
+ *   may be used to endorse or promote products derived from this software
+ *   without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.jmonkeyengine.screenshottests.testframework;
+
+import com.jme3.math.ColorRGBA;
+
+/**
+ * @author Richard Tingle (aka richtea)
+ */
+public enum PixelSamenessDegree{
+    SAME(1, null),
+    NEGLIGIBLY_DIFFERENT(1, ColorRGBA.Green),
+    SUBTLY_DIFFERENT(10, ColorRGBA.Blue),
+
+    MEDIUMLY_DIFFERENT(20, ColorRGBA.Yellow),
+
+    VERY_DIFFERENT(60,ColorRGBA.Orange),
+
+    EXTREMELY_DIFFERENT(100,ColorRGBA.Red);
+
+    private final int maximumAllowedDifference;
+
+    private final ColorRGBA colorInDebugImage;
+
+    PixelSamenessDegree(int maximumAllowedDifference, ColorRGBA colorInDebugImage){
+        this.colorInDebugImage = colorInDebugImage;
+        this.maximumAllowedDifference = maximumAllowedDifference;
+    }
+
+    public ColorRGBA getColorInDebugImage(){
+        return colorInDebugImage;
+    }
+
+    public int getMaximumAllowedDifference(){
+        return maximumAllowedDifference;
+    }
+}