action.yaml

name: 'Visual Regression Testing with Backstop.js'
description: 'Runs visual regression testing with backstop.js between a production site and PR version of a site'
inputs:
  baseline_url:  # owner of the repo
    description: 'The production URL of the site you want to use as the baseline.'
    required: true
  test_url:
    description: 'The URL of the PR site we want to test against the baseline'
    required: true
  report_results:
    description: "Do you want this activity to report the results back to the calling workflow (true) or pass/fail directly (false)? Default is false."
    required: false
    default: 'false'
  paths_location:
    description: "Path to the template-paths.js file. Default is ./.github/tests/vrt/ from the template's repo root"
    default: "./.github/tests/vrt/"
  threshold:
    description: 'Percentage of different pixels allowed to pass test. NOT CURRENTLY IMPLEMENTED.'
    required: false
    default: "0.1"
  baseline_url_response:
    description: 'The HTTP response status code we expect from the server for the baseline URL. Defaults to 200'
    required: false
    default: 200
  test_url_response:
    description: 'The HTTP response status code we expect from the server for the test URL. Defaults to 200'
    required: false
    default: 200
  baseline_url_insecure:
    description: 'Should we use the --insecure flag with curl when testing the baseline URL? Defaults to false'
    required: false
    default: 'false'
  test_url_insecure:
    description: 'Should we use the --insecure flag with curl when testing the test URL? Defaults to false'
    required: false
    default: 'false'
outputs:
  results:
    description: "Reports from VRT"
    value: ${{ steps.report-results.outputs.results }}

# Things we need to do
# Verify the URLs we were given respond
# Install backstop
# Pass the reference url to backstop
# Run npm run-script backstop:reference
# Pass the test url to backstop
# Run npm run-script backstop:test
# if test fails store the test results as an artifact and report back `false`
# if test passes, report back `true`
runs:
  using: "composite"
  steps:
    # Test that the Baseline/Reference URL returns a 200
    - name: 'Test Baseline URL'
      id: test-baseline-url
      uses: platformsh/gha-check-url@main
      with:
        test_url: ${{ inputs.baseline_url }}
        http_response: ${{ inputs.baseline_url_response }}
        allow_insecure: ${{ inputs.baseline_url_insecure }}
    #
    # Test that the URL for the site to test returns a 200
    # @todo since this is a repeat of the previous step, consider making it an action we can reuse
    - name: 'Test Testing URL'
      id: verify-repo
      uses: platformsh/gha-check-url@main
      with:
        test_url: ${{ inputs.test_url }}
        http_response: ${{ inputs.test_url_response }}
        allow_insecure: ${{ inputs.test_url_insecure }}
    #
    # Check that what the calling workflow gave us for report_results is something we can use
    - name: 'Verify return value'
      id: 'verify-return-vale'
      shell: bash
      run: |
        # we need to make sure they gave us a true/false, yes/no
        if [[ "${{ inputs.report_results }}" =~ "yes" ]] || [[ "${{ inputs.report_results }}" =~ "true" ]]; then
          echo "report_results=true" >> $GITHUB_ENV  
        else
          echo "report_results=false" >> $GITHUB_ENV
        fi
    #
    # Determine where extra scenarios are possibly stored
    - name: 'Set path file location'
      id: path-file-location
      shell: bash
      run: |
        # check to see if we have a relative path from the calling repo root
        if [[ "${{inputs.paths_location}}" =~ ^\./ ]]; then
          echo "::notice::Path location is relative to repo root. Setting full path"
          localRelative="${{ inputs.paths_location }}"
          # and let's remove the ./ since we're creating a full path
          path_location="${GITHUB_WORKSPACE}/${localRelative:2}"
        else 
          echo "::notice::Path location is already absolute. No need to adjust"
          path_location="${{ inputs.paths_location }}"
        fi
        
        # now we need to see if we're missing the trailing directory separator
        if [[ "${path_location}" =~ [^/]$ ]]; then
          path_location="${path_location}/"
        fi
        echo "::notice::Saving path location as ${path_location}"
        echo "paths_location=${path_location}" >> $GITHUB_ENV
    #
    # Install Backstop
    - name: 'install backstop'
      id: install-backstop
      shell: bash
      working-directory: ${{ github.action_path }}
      run: |
        npm install
    #
    # Get the calling project's repository so we can check it for extra scenarios
    - uses: actions/checkout@v3
    #
    # Instruct backstop to build our references
    - name: 'build reference'
      id: build-reference
      shell: bash
      working-directory: ${{ github.action_path }}
      run: |
        echo "::notice::Starting backstopjs to generate baseline images using ${{inputs.baseline_url}}."
        suppress=$(npm run-script backstop:reference -- --refURL="${{inputs.baseline_url}}" --testURL="${{inputs.test_url}}" --testPath="${{env.paths_location}}")
    #
    # Now we actually do the visual regression test
    - name: 'run test'
      id: run-test
      continue-on-error: true
      shell: bash
      working-directory: ${{ github.action_path }}
      run: |
        set +e
        echo "::notice::Starting Backstop VRT testing against ${{inputs.test_url}}"
        suppress=$(npm run-script backstop:test -- --refURL="${{inputs.baseline_url}}" --testURL="${{inputs.test_url}}" --testPath="${{env.paths_location}}")
        testresults=$?
        echo "::notice::Visual Regression Testing complete. Preparing results..."
        
        if (( 0 != $testresults )); then
          echo "::error::Backstop visual regression testing failed. Please see the generated report"
          results="false"
        else
          echo "::notice::Backstop visual regression testing passed."
          results="true"
        fi
      
        echo "vrt_results=${results}" >> $GITHUB_ENV
    #
    # If the VRT failed, export the test results as an artifact
    - name: 'export reports'
      id: export-reports
      if: ${{ env.vrt_results == 'false' }}
      uses: actions/upload-artifact@v4
      with:
        name: vrt-report
        path: |
          ${{ github.action_path }}/backstop_data
          !${{ github.action_path }}/backstop_data/engine_scripts/*
    #
    # Return the results or send an exit code based on the VRT
    - name: 'Report Results'
      id: 'report-results'
      shell: bash
      run: |
        # set the output
        echo "results=${{ env.vrt_results }}" >> $GITHUB_OUTPUT
        
        if [[ "false" == "${{ env.report_results }}" ]]; then
          # they want the action result to be the same as the VRT result
          exitcode=$([ "true" = "${{ env.vrt_results }}" ]);
          exit ${exitcode}
        fi