From 141c208037675652cec52ddb5f084b1c39d6fd5b Mon Sep 17 00:00:00 2001 From: Howard Edwards Date: Thu, 3 Oct 2024 15:59:18 -0400 Subject: [PATCH] Expand sample test content to handle more scenarios (#1224) Address #759 --- client/components/DisclaimerInfo/index.jsx | 1 - .../e2e/snapshots/saved/_data-management.html | 194 +- .../saved/_report_67_targets_20.html | 3771 +++++++++++++++++ .../tests/e2e/snapshots/saved/_reports.html | 92 + .../snapshots/saved/_test-plan-report_1.html | 121 +- .../e2e/snapshots/saved/_test-queue.html | 29 +- client/tests/e2e/snapshots/utils.js | 3 +- server/scripts/populate-test-data/index.js | 182 +- .../populate-test-data/pg_dump_test_data.sql | 25 +- .../populateFakeTestResults.js | 211 +- .../tests/integration/dataManagement.test.js | 388 +- server/tests/integration/testQueue.test.js | 282 +- .../services/TestPlanRunService.test.js | 2 +- 13 files changed, 4698 insertions(+), 603 deletions(-) create mode 100644 client/tests/e2e/snapshots/saved/_report_67_targets_20.html diff --git a/client/components/DisclaimerInfo/index.jsx b/client/components/DisclaimerInfo/index.jsx index 5730e5433..79964d9c3 100644 --- a/client/components/DisclaimerInfo/index.jsx +++ b/client/components/DisclaimerInfo/index.jsx @@ -104,7 +104,6 @@ const DisclaimerInfo = ({ phase }) => {
setExpanded(!expanded)} diff --git a/client/tests/e2e/snapshots/saved/_data-management.html b/client/tests/e2e/snapshots/saved/_data-management.html index cff73b369..6950c45da 100644 --- a/client/tests/e2e/snapshots/saved/_data-management.html +++ b/client/tests/e2e/snapshots/saved/_data-management.html @@ -276,11 +276,11 @@

+
@@ -377,7 +377,7 @@

Test Plans Status Summary

type="button" aria-pressed="false" class="css-14jv778 btn btn-secondary"> - R&D Complete (33) + R&D Complete (32)
  • @@ -401,7 +401,7 @@

    Test Plans Status Summary

    type="button" aria-pressed="false" class="css-14jv778 btn btn-secondary"> - Recommended Plans (1) + Recommended Plans (2)
    • @@ -482,6 +482,121 @@

    Test Plans Status Summary

    + + Action Menu Button Example Using aria-activedescendant + + +
    + JAWS, NVDA and VoiceOver for macOS +
    + + +
    + Recommended +

    Since Sep 23, 2024

    +
    + + N/A + +
    + V24.09.18Review Completed Sep 22, 2024 +
    + + +
    + V24.09.18Review Completed Sep 23, 2024 +
    + + +
    + V24.09.18 +
    Approved Sep 23, 2024 +
    + + + Checkbox Example (Tri State)Test Plans Status Summary - + Modal Dialog ExampleTest Plans Status Summary None Yet - + Alert Example @@ -790,7 +905,7 @@

    Test Plans Status Summary

    Not Started None Yet - + Command Button ExampleTest Plans Status Summary Not Started None Yet - + Select Only Combobox ExampleTest Plans Status Summary Not Started None Yet - + Toggle ButtonTest Plans Status Summary Not Started None Yet - - - Action Menu Button Example Using aria-activedescendant - - -
    - JAWS, NVDA and VoiceOver for macOS -
    - - -
    - R&D -

    Complete Sep 18, 2024

    -
    - - -
    - V24.09.18 -
    - - Not Started - Not Started - None Yet - + + + + + + + + + NVDA 2023.3.3 and Chrome for Checkbox Example (Mixed-State) | + ARIA-AT Reports + + + +
    +
    + +
    +
    +
    +

    + Checkbox Example (Mixed-State) with NVDA 2023.3.3 and + Chrome +

    + +

    Introduction

    +

    + This page shows detailed results for each test, including individual + commands that the tester entered into the AT, the output which was + captured from the AT in response, and whether that output was deemed + passing or failing for each of the assertions. The open test button + next to each test allows you to preview the test in your browser. +

    +

    Metadata

    + +

    NVDA Versions Summary

    +

    + The following table displays a summary of data for all versions of + NVDA that have been tested. +

    +
    + + + + + + + + + + + + + + + + + + + + + + + +
    VersionsMust-Have BehaviorsShould-Have BehaviorsMay-Have Behaviors
    2023.3.3124 of 124 passed35 of 36 passedNone
    2020.4124 of 124 passed36 of 36 passedNone
    +
    +

    Results for NVDA 2023.3.3 and Chrome

    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Test NameMust-Have BehaviorsShould-Have BehaviorsMay-Have Behaviors
    + Navigate forwards to a mixed checkbox in reading mode + 24 of 24 passed4 of 4 passedNone
    + Navigate forwards to a mixed checkbox in interaction + mode + 6 of 6 passed1 of 1 passedNone
    + Navigate backwards to a mixed checkbox in reading mode + 24 of 24 passed3 of 4 passedNone
    + Navigate backwards to a mixed checkbox in interaction + mode + 6 of 6 passed1 of 1 passedNone
    + Read information about a mixed checkbox in reading + mode + 8 of 8 passed6 of 6 passedNone
    + Read information about a mixed checkbox in interaction + mode + 8 of 8 passed6 of 6 passedNone
    + Operate a mixed checkbox in reading mode + 2 of 2 passed1 of 1 passedNone
    + Operate a mixed checkbox in interaction mode + 2 of 2 passed1 of 1 passedNone
    + Operate an unchecked checkbox in reading mode + 2 of 2 passed1 of 1 passedNone
    + Operate an unchecked checkbox in interaction mode + 2 of 2 passed1 of 1 passedNone
    + Navigate forwards out of a checkbox group in reading + mode + 16 of 16 passed4 of 4 passedNone
    + Navigate forwards out of a checkbox group in interaction + mode + 4 of 4 passed1 of 1 passedNone
    + Navigate backwards out of a checkbox group in reading + mode + 16 of 16 passed4 of 4 passedNone
    + Navigate backwards out of a checkbox group in interaction + mode + 4 of 4 passed1 of 1 passedNone
    All Tests124 of 124 passed35 of 36 passedNone
    +
    +
    +

    + Test 1: Navigate forwards to a mixed checkbox in reading + mode (28 passed, 0 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    X Results: 7 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + X Results +
    PriorityAssertionResult
    MUSTRole 'group' is conveyedPassed
    MUSTName 'Sandwich Condiments' is conveyedPassed
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName ' All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None + +

    F Results: 7 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + F Results +
    PriorityAssertionResult
    MUSTRole 'group' is conveyedPassed
    MUSTName 'Sandwich Condiments' is conveyedPassed
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName ' All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None + +

    Tab Results: 7 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Tab Results +
    PriorityAssertionResult
    MUSTRole 'group' is conveyedPassed
    MUSTName 'Sandwich Condiments' is conveyedPassed
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName ' All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None + +

    + Down Arrow then Down Arrow Results: 7 passed, 0 + failed +

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Down Arrow then Down Arrow Results +
    PriorityAssertionResult
    MUSTRole 'group' is conveyedPassed
    MUSTName 'Sandwich Condiments' is conveyedPassed
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName ' All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +

    + Test 2: Navigate forwards to a mixed checkbox in interaction + mode (7 passed, 0 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    Tab Results: 7 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Tab Results +
    PriorityAssertionResult
    MUSTRole 'group' is conveyedPassed
    MUSTName 'Sandwich Condiments' is conveyedPassed
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName ' All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +

    + Test 3: Navigate backwards to a mixed checkbox in reading + mode (27 passed, 1 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    Shift+X Results: 6 passed, 1 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Shift+X Results +
    PriorityAssertionResult
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Failed
    MUSTRole 'group' is conveyedPassed
    MUSTName 'Sandwich Condiments' is conveyedPassed
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName 'All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: +
    + + + + + + + + + + + + + + + +
    BehaviorDetailsImpact
    OtherSeeded other unexpected behaviorMODERATE
    +
    + +

    Shift+F Results: 7 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Shift+F Results +
    PriorityAssertionResult
    MUSTRole 'group' is conveyedPassed
    MUSTName 'Sandwich Condiments' is conveyedPassed
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName 'All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None + +

    Shift+Tab Results: 7 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Shift+Tab Results +
    PriorityAssertionResult
    MUSTRole 'group' is conveyedPassed
    MUSTName 'Sandwich Condiments' is conveyedPassed
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName 'All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None + +

    + Up Arrow then Up Arrow Results: 7 passed, 0 failed +

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Up Arrow then Up Arrow Results +
    PriorityAssertionResult
    MUSTRole 'group' is conveyedPassed
    MUSTName 'Sandwich Condiments' is conveyedPassed
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName 'All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +

    + Test 4: Navigate backwards to a mixed checkbox in interaction + mode (7 passed, 0 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    Shift+Tab Results: 7 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Shift+Tab Results +
    PriorityAssertionResult
    MUSTRole 'group' is conveyedPassed
    MUSTName 'Sandwich Condiments' is conveyedPassed
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName 'All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +

    + Test 5: Read information about a mixed checkbox in reading + mode (14 passed, 0 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    Insert+Tab Results: 7 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Insert+Tab Results +
    PriorityAssertionResult
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName 'All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULDRole 'group' is conveyedPassed
    SHOULDName 'Sandwich Condiments' is conveyedPassed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None + +

    Insert+Up Results: 7 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Insert+Up Results +
    PriorityAssertionResult
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName 'All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULDRole 'group' is conveyedPassed
    SHOULDName 'Sandwich Condiments' is conveyedPassed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +

    + Test 6: Read information about a mixed checkbox in interaction + mode (14 passed, 0 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    Insert+Tab Results: 7 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Insert+Tab Results +
    PriorityAssertionResult
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName 'All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULDRole 'group' is conveyedPassed
    SHOULDName 'Sandwich Condiments' is conveyedPassed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None + +

    Insert+Up Results: 7 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Insert+Up Results +
    PriorityAssertionResult
    MUSTRole 'checkbox' is conveyedPassed
    MUSTName 'All condiments' is conveyedPassed
    MUSTState 'mixed' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULDRole 'group' is conveyedPassed
    SHOULDName 'Sandwich Condiments' is conveyedPassed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +

    + Test 7: Operate a mixed checkbox in reading + mode (3 passed, 0 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    Space Results: 3 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Space Results +
    PriorityAssertionResult
    MUSTChange in state, to 'checked', is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +

    + Test 8: Operate a mixed checkbox in interaction + mode (3 passed, 0 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    Space Results: 3 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Space Results +
    PriorityAssertionResult
    MUSTChange in state, to 'checked', is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +

    + Test 9: Operate an unchecked checkbox in reading + mode (3 passed, 0 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    Space Results: 3 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Space Results +
    PriorityAssertionResult
    MUSTChange in state, to 'mixed', is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +

    + Test 10: Operate an unchecked checkbox in interaction + mode (3 passed, 0 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    Space Results: 3 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Space Results +
    PriorityAssertionResult
    MUSTChange in state, to 'mixed', is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +

    + Test 11: Navigate forwards out of a checkbox group in reading + mode (20 passed, 0 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    Down Arrow Results: 5 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Down Arrow Results +
    PriorityAssertionResult
    MUSTGroup boundary is conveyedPassed
    MUSTRole 'link' is conveyedPassed
    MUSTName 'Navigate backwards from here' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None + +

    U Results: 5 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + U Results +
    PriorityAssertionResult
    MUSTGroup boundary is conveyedPassed
    MUSTRole 'link' is conveyedPassed
    MUSTName 'Navigate backwards from here' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None + +

    K Results: 5 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + K Results +
    PriorityAssertionResult
    MUSTGroup boundary is conveyedPassed
    MUSTRole 'link' is conveyedPassed
    MUSTName 'Navigate backwards from here' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None + +

    Tab Results: 5 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Tab Results +
    PriorityAssertionResult
    MUSTGroup boundary is conveyedPassed
    MUSTRole 'link' is conveyedPassed
    MUSTName 'Navigate backwards from here' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +

    + Test 12: Navigate forwards out of a checkbox group in interaction + mode (5 passed, 0 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    Tab Results: 5 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Tab Results +
    PriorityAssertionResult
    MUSTGroup boundary is conveyedPassed
    MUSTRole 'link' is conveyedPassed
    MUSTName 'Navigate backwards from here' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +

    + Test 13: Navigate backwards out of a checkbox group in reading + mode (20 passed, 0 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    + Up Arrow then Up Arrow Results: 5 passed, 0 failed +

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Up Arrow then Up Arrow Results +
    PriorityAssertionResult
    MUSTGroup boundary is conveyedPassed
    MUSTRole 'link' is conveyedPassed
    MUSTName 'Navigate forwards from here' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None + +

    Shift+K Results: 5 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Shift+K Results +
    PriorityAssertionResult
    MUSTGroup boundary is conveyedPassed
    MUSTRole 'link' is conveyedPassed
    MUSTName 'Navigate forwards from here' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None + +

    Shift+U Results: 5 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Shift+U Results +
    PriorityAssertionResult
    MUSTGroup boundary is conveyedPassed
    MUSTRole 'link' is conveyedPassed
    MUSTName 'Navigate forwards from here' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None + +

    Shift+Tab Results: 5 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Shift+Tab Results +
    PriorityAssertionResult
    MUSTGroup boundary is conveyedPassed
    MUSTRole 'link' is conveyedPassed
    MUSTName 'Navigate forwards from here' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +

    + Test 14: Navigate backwards out of a checkbox group in interaction + mode (5 passed, 0 failed) +
    +
    + + The information in this report is generated from recommended + tests. Recommended ARIA-AT tests have been reviewed by + assistive technology developers and represent consensus + regarding: applicability and validity of the tests, and + accuracy of test results. +
    +
    +

    +
    + Raise an IssueOpen Test +
    +
    +
    +

    Results for each command

    +

    Shift+Tab Results: 5 passed, 0 failed

    +

    NVDA Response:

    +
    + automatically seeded sample output +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Shift+Tab Results +
    PriorityAssertionResult
    MUSTGroup boundary is conveyedPassed
    MUSTRole 'link' is conveyedPassed
    MUSTName 'Navigate forwards from here' is conveyedPassed
    MUST + Other behaviors that create severe negative impacts are + not exhibited + Passed
    SHOULD + Other behaviors that create moderate negative impacts are + not exhibited + Passed
    +
    + Other behaviors that create negative impact: None +
    +
    +

    + +

    +
    + +
    +
    +
    +
    + +
    + + + diff --git a/client/tests/e2e/snapshots/saved/_reports.html b/client/tests/e2e/snapshots/saved/_reports.html index 21dc0c932..4097fc174 100644 --- a/client/tests/e2e/snapshots/saved/_reports.html +++ b/client/tests/e2e/snapshots/saved/_reports.html @@ -93,12 +93,98 @@

    Support Levels

    Test Plan + JAWS and Firefox JAWS and Chrome + NVDA and Firefox NVDA and Chrome + VoiceOver for macOS and Chrome VoiceOver for macOS and Safari + + + Action Menu Button Example Using aria-activedescendant + + +
    +
    + 100% +
    +
    100%
    +
    + + +
    +
    + 100% +
    +
    100%
    +
    + + +
    +
    + 100% +
    +
    100%
    +
    + + +
    +
    + 100% +
    +
    100%
    +
    + + +
    +
    + 100% +
    +
    100%
    +
    + + +
    +
    + 100% +
    +
    100%
    +
    + + Support Levels >Recommended + None
    @@ -121,6 +208,7 @@

    Support Levels

     + None
    @@ -133,6 +221,7 @@

    Support Levels

     + None
    @@ -156,6 +245,7 @@

    Support Levels

    >Candidate + None
    @@ -168,6 +258,7 @@

    Support Levels

     + None
    @@ -180,6 +271,7 @@

    Support Levels

     + None
    diff --git a/client/tests/e2e/snapshots/saved/_test-plan-report_1.html b/client/tests/e2e/snapshots/saved/_test-plan-report_1.html index 333d8d1e9..a4caaaa31 100644 --- a/client/tests/e2e/snapshots/saved/_test-plan-report_1.html +++ b/client/tests/e2e/snapshots/saved/_test-plan-report_1.html @@ -122,13 +122,13 @@
      -
    1. +
    2. Navigate forwards to a not pressed toggle button in reading mode + title="Has Conflicts">
      4. -
    4. +
    5. Navigate backwards to a pressed toggle button in reading mode + title="Has Conflicts">
      6. -
    6. +
    7. Navigate forwards to a pressed toggle button in interaction mode + title="Has Conflicts">
      8. -
    8. +
    9. Navigate backwards to a pressed toggle button in interaction mode + title="Has Conflicts">
    +
    @@ -1403,7 +1423,7 @@

    Test Options

    Test Options
    -
    +
    +
    +

    + Review Conflicts for "Navigate forwards to a not pressed + toggle button in reading mode" +

    +
      +
    1. +

      + Assertion Results for "Down Arrow" Command and "Role + 'button' is conveyed" Assertion +

      +
        +
      • + Tester esmeralda-baggins recorded output + "automatically seeded sample output" and marked + assertion as passing. +
        • +
      • + Tester tom-proudfeet recorded output + "automatically seeded sample output" and marked + assertion as failing. +
        • +
      +
      2. +
    2. +

      + Assertion Results for "B" Command and "Role 'button' + is conveyed" Assertion +

      +
        +
      • + Tester esmeralda-baggins recorded output + "automatically seeded sample output" and marked + assertion as passing. +
        • +
      • + Tester tom-proudfeet recorded output + "automatically seeded sample output" and marked + assertion as failing. +
        • +
      +
      3. +
    3. +

      + Assertion Results for "F" Command and "Role 'button' + is conveyed" Assertion +

      +
        +
      • + Tester esmeralda-baggins recorded output + "automatically seeded sample output" and marked + assertion as passing. +
        • +
      • + Tester tom-proudfeet recorded output + "automatically seeded sample output" and marked + assertion as failing. +
        • +
      +
      4. +
    4. +

      + Assertion Results for "Tab" Command and "Role + 'button' is conveyed" Assertion +

      +
        +
      • + Tester esmeralda-baggins recorded output + "automatically seeded sample output" and marked + assertion as passing. +
        • +
      • + Tester tom-proudfeet recorded output + "automatically seeded sample output" and marked + assertion as failing. +
        • +
      +
      5. +
    +
    +
    diff --git a/client/tests/e2e/snapshots/saved/_test-queue.html b/client/tests/e2e/snapshots/saved/_test-queue.html index 5bc5eac76..a80adbe7c 100644 --- a/client/tests/e2e/snapshots/saved/_test-queue.html +++ b/client/tests/e2e/snapshots/saved/_test-queue.html @@ -205,6 +205,9 @@

    - @@ -1673,6 +1676,15 @@

    >esmeralda-baggins12 of 16 tests complete +
    +
  • + tom-proudfeet13 of 16 tests complete +
    • @@ -1680,14 +1692,15 @@

    + style="clip-path: inset(0px 0px 0px 78%)">
    75% complete by esmeralda-baggins 78% complete by 2 testers with 14 conflicts diff --git a/client/tests/e2e/snapshots/utils.js b/client/tests/e2e/snapshots/utils.js index 9984eefc0..749a0963b 100644 --- a/client/tests/e2e/snapshots/utils.js +++ b/client/tests/e2e/snapshots/utils.js @@ -14,7 +14,8 @@ const snapshotRoutes = [ '/run/2', '/data-management/meter', '/candidate-test-plan/24/1', - '/test-queue/2/conflicts' + '/test-queue/2/conflicts', + '/report/67/targets/20' ]; async function cleanAndNormalizeSnapshot(page) { diff --git a/server/scripts/populate-test-data/index.js b/server/scripts/populate-test-data/index.js index 1d43c0d3c..92da3f641 100644 --- a/server/scripts/populate-test-data/index.js +++ b/server/scripts/populate-test-data/index.js @@ -1,7 +1,10 @@ /* eslint-disable no-console */ const fs = require('fs'); const { sequelize } = require('../../models'); -const populateFakeTestResults = require('./populateFakeTestResults'); +const { + populateFakeTestResults, + FAKE_RESULT_CONFLICTS_OPTIONS +} = require('./populateFakeTestResults'); const populateTestDatabase = async transaction => { const testDataScript = fs.readFileSync( @@ -11,23 +14,8 @@ const populateTestDatabase = async transaction => { await sequelize.query(testDataScript, { transaction }); - await populateFakeTestResults( - 1, - [ - 'completeAndPassing', - 'incompleteAndPassing', - 'incompleteAndFailingDueToNoOutputAssertions', - 'incompleteAndEmpty', - null, - 'completeAndFailingDueToIncorrectAssertions', - 'completeAndFailingDueToNoOutputAssertions', - 'completeAndFailingDueToUnexpectedBehaviors', - 'completeAndPassing' - ], - { transaction } - ); - - // Report 2 contains conflicts which is convenient to have in sample data + // Convenient way to generate conflicts between TestPlanRuns attached to + // the same TestPlanReport const conflicts = [ ['completeAndPassing', 'completeAndFailingDueToIncorrectAssertions'], [ @@ -41,6 +29,24 @@ const populateTestDatabase = async transaction => { ] ]; + await populateFakeTestResults( + 1, + [ + 'completeAndPassing', + 'incompleteAndPassing', + 'incompleteAndFailingDueToNoOutputAssertions', + 'incompleteAndEmpty', + null, + 'completeAndFailingDueToIncorrectAssertions', + 'completeAndFailingDueToNoOutputAssertions', + 'completeAndFailingDueToUnexpectedBehaviors' + ], + { + transaction, + numFakeTestResultConflicts: FAKE_RESULT_CONFLICTS_OPTIONS.ALL + } + ); + await populateFakeTestResults( 2, [ @@ -51,7 +57,7 @@ const populateTestDatabase = async transaction => { conflicts[1][0], conflicts[2][0], conflicts[3][0], - 'incompleteAndPassing' + 'incompleteAndFailingDueToUnexpectedBehaviors' ], { transaction } ); @@ -66,10 +72,7 @@ const populateTestDatabase = async transaction => { conflicts[1][1], conflicts[2][1], conflicts[3][1], - 'incompleteAndFailingDueToUnexpectedBehaviors', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing' + 'incompleteAndFailingDueToUnexpectedBehaviors' ], { transaction } ); @@ -81,13 +84,7 @@ const populateTestDatabase = async transaction => { 'completeAndPassing', 'completeAndFailingDueToIncorrectAssertions', 'completeAndFailingDueToNoOutputAssertions', - 'completeAndFailingDueToUnexpectedBehaviors', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing' + 'completeAndFailingDueToUnexpectedBehaviors' ], { transaction } ); @@ -101,8 +98,7 @@ const populateTestDatabase = async transaction => { 'completeAndPassing', 'completeAndFailingDueToNoOutputAssertions', 'completeAndPassing', - 'completeAndFailingDueToUnexpectedBehaviors', - 'completeAndPassing' + 'completeAndFailingDueToUnexpectedBehaviors' ], { transaction } ); @@ -115,8 +111,7 @@ const populateTestDatabase = async transaction => { 'completeAndFailingDueToIncorrectAssertions', 'completeAndPassing', 'completeAndFailingDueToIncorrectAssertions', - 'completeAndFailingDueToUnexpectedBehaviors', - 'completeAndPassing' + 'completeAndFailingDueToUnexpectedBehaviors' ], { transaction } ); @@ -129,103 +124,98 @@ const populateTestDatabase = async transaction => { 'completeAndPassing', 'completeAndFailingDueToIncorrectAssertions', 'completeAndPassing', - 'completeAndFailingDueToNoOutputAssertions', - 'completeAndPassing' + 'completeAndFailingDueToNoOutputAssertions' ], { transaction } ); - await populateFakeTestResults( - 8, - [ - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing' - ], - { transaction } - ); + await populateFakeTestResults(8, ['completeAndPassing'], { transaction }); - await populateFakeTestResults( - 9, - [ - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing' - ], - { transaction } - ); + await populateFakeTestResults(9, ['completeAndPassing'], { transaction }); + + await populateFakeTestResults(10, ['completeAndPassing'], { transaction }); + + await populateFakeTestResults(11, ['completeAndPassing'], { transaction }); + + await populateFakeTestResults(12, ['completeAndPassing'], { transaction }); + + await populateFakeTestResults(13, ['completeAndPassing'], { transaction }); + + await populateFakeTestResults(14, ['completeAndPassing'], { + transaction + }); + + await populateFakeTestResults(15, ['completeAndPassing'], { + transaction + }); + + await populateFakeTestResults(16, ['completeAndPassing'], { + transaction + }); + + await populateFakeTestResults(17, ['completeAndPassing'], { + transaction + }); + + await populateFakeTestResults(18, ['completeAndPassing'], { + transaction + }); + + await populateFakeTestResults(19, ['completeAndPassing'], { + transaction + }); + // Slightly different from older already recommended TestPlanRun 13 to get + // multiple results shown for: + // Recommended Report + Checkbox Example (Mixed-State) + NVDA + Chrome await populateFakeTestResults( - 10, + 20, [ 'completeAndPassing', 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing' + 'completeAndFailingDueToUnexpectedBehaviors' ], { transaction } ); + // Conflicting with TestPlanRun 1 await populateFakeTestResults( - 11, + 21, [ + 'completeAndFailingDueToIncorrectAssertions', + 'incompleteAndPassing', + 'incompleteAndFailingDueToNoOutputAssertions', + 'incompleteAndEmpty', 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing', - 'completeAndPassing' + 'completeAndFailingDueToUnexpectedBehaviors' ], - { transaction } - ); - - await populateFakeTestResults( - 12, - ['completeAndPassing', 'completeAndPassing', 'completeAndPassing'], - { transaction } - ); - - await populateFakeTestResults( - 13, - ['completeAndPassing', 'completeAndPassing', 'completeAndPassing'], - { transaction } + { + transaction, + numFakeTestResultConflicts: FAKE_RESULT_CONFLICTS_OPTIONS.ALL + } ); - await populateFakeTestResults(14, new Array(16).fill('completeAndPassing'), { + await populateFakeTestResults(22, ['completeAndPassing'], { transaction }); - await populateFakeTestResults(15, new Array(8).fill('completeAndPassing'), { + await populateFakeTestResults(23, ['completeAndPassing'], { transaction }); - await populateFakeTestResults(16, new Array(3).fill('completeAndPassing'), { + await populateFakeTestResults(24, ['completeAndPassing'], { transaction }); - await populateFakeTestResults(17, new Array(3).fill('completeAndPassing'), { + await populateFakeTestResults(25, ['completeAndPassing'], { transaction }); - await populateFakeTestResults(18, new Array(3).fill('completeAndPassing'), { + await populateFakeTestResults(26, ['completeAndPassing'], { transaction }); - await populateFakeTestResults(19, new Array(12).fill('completeAndPassing'), { + await populateFakeTestResults(27, ['completeAndPassing'], { transaction }); diff --git a/server/scripts/populate-test-data/pg_dump_test_data.sql b/server/scripts/populate-test-data/pg_dump_test_data.sql index 7768e440f..7c6ec522d 100644 --- a/server/scripts/populate-test-data/pg_dump_test_data.sql +++ b/server/scripts/populate-test-data/pg_dump_test_data.sql @@ -83,6 +83,14 @@ INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinal INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "minimumAtVersionId", "browserId", "vendorReviewStatus") VALUES (17, get_test_plan_version_id(text 'Command Button Example', '2'), '2023-12-13 14:18:23.602-05', '2023-12-14', 2, 2, 2, 'READY'); INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "minimumAtVersionId", "browserId", "vendorReviewStatus") VALUES (18, get_test_plan_version_id(text 'Command Button Example', '2'), '2023-12-13 14:18:23.602-05', '2023-12-14', 3, 3, 3, 'READY'); INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "atId", "minimumAtVersionId", "browserId", "vendorReviewStatus") VALUES (19, get_test_plan_version_id(text 'Modal Dialog Example', '2'), '2024-05-14 14:18:23.602-05', 2, 2, 2, 'READY'); +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (20, get_test_plan_version_id(text 'Checkbox Example (Mixed-State)', '1'), '2021-05-14 14:18:23.602-05', '2022-08-07', 2, 4, 2, 'READY'); +-- Fully recommended TestPlanVersion +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (21, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 1, 1, 1, 'READY'); +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (22, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 1, 1, 2, 'READY'); +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (23, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 2, 4, 1, 'READY'); +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (24, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 2, 4, 2, 'READY'); +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (25, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 3, 5, 2, 'READY'); +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (26, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 3, 5, 3, 'READY'); -- -- Data for Name: TestPlanVersion; Type: TABLE DATA; Schema: public; Owner: atr @@ -91,10 +99,11 @@ INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "atId", "min UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2022-07-06' WHERE id = get_test_plan_version_id(text 'Toggle Button', '1'); UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2022-07-06' WHERE id = get_test_plan_version_id(text 'Select Only Combobox Example', '1'); UPDATE "TestPlanVersion" SET "phase" = 'CANDIDATE', "draftPhaseReachedAt" = '2022-07-06', "candidatePhaseReachedAt" = '2022-07-06', "recommendedPhaseTargetDate" = '2023-01-02' WHERE id = get_test_plan_version_id(text 'Modal Dialog Example', '1'); -UPDATE "TestPlanVersion" SET "phase" = 'RECOMMENDED', "candidatePhaseReachedAt" = '2022-07-06', "recommendedPhaseTargetDate" = '2023-01-02', "recommendedPhaseReachedAt" = '2023-01-03' WHERE id = get_test_plan_version_id(text 'Checkbox Example (Mixed-State)', '1'); +UPDATE "TestPlanVersion" SET "phase" = 'RECOMMENDED', "draftPhaseReachedAt" = '2022-07-07', "candidatePhaseReachedAt" = '2022-07-06', "recommendedPhaseTargetDate" = '2023-01-02', "recommendedPhaseReachedAt" = '2023-01-03' WHERE id = get_test_plan_version_id(text 'Checkbox Example (Mixed-State)', '1'); UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2022-07-06' WHERE id = get_test_plan_version_id(text 'Alert Example', '1'); UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2023-12-14' WHERE id = get_test_plan_version_id(text 'Command Button Example', '2'); UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2024-05-14' WHERE id = get_test_plan_version_id(text 'Modal Dialog Example', '2'); +UPDATE "TestPlanVersion" SET "phase" = 'RECOMMENDED', "draftPhaseReachedAt" = '2024-09-20', "candidatePhaseReachedAt" = '2024-09-22', "recommendedPhaseTargetDate" = '2025-03-21', "recommendedPhaseReachedAt" = '2024-09-23' WHERE id = get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'); -- -- Data for Name: User; Type: TABLE DATA; Schema: public; Owner: atr @@ -102,13 +111,17 @@ UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2024-05 INSERT INTO "User" (id, username, "createdAt", "updatedAt") VALUES (1, 'esmeralda-baggins', '2021-05-14 13:57:16.232-05', '2021-05-14 13:57:20.473-05'); INSERT INTO "User" (id, username, "createdAt", "updatedAt") VALUES (2, 'tom-proudfeet', '2021-05-14 13:57:16.232-05', '2021-05-14 13:57:20.473-05'); +INSERT INTO "User" (id, username, "createdAt", "updatedAt") VALUES (3, 'foo-bar', '2021-05-14 13:57:16.232-05', '2021-05-14 13:57:20.473-05'); INSERT INTO "UserAts" ("userId", "atId") VALUES (1, 1); INSERT INTO "UserAts" ("userId", "atId") VALUES (1, 2); INSERT INTO "UserAts" ("userId", "atId") VALUES (2, 3); +INSERT INTO "UserAts" ("userId", "atId") VALUES (3, 1); +INSERT INTO "UserAts" ("userId", "atId") VALUES (3, 2); +INSERT INTO "UserAts" ("userId", "atId") VALUES (3, 3); INSERT INTO "UserRoles" ("userId", "roleName") VALUES (1, 'ADMIN'); INSERT INTO "UserRoles" ("userId", "roleName") VALUES (1, 'TESTER'); INSERT INTO "UserRoles" ("userId", "roleName") VALUES (2, 'TESTER'); - +INSERT INTO "UserRoles" ("userId", "roleName") VALUES (3, 'TESTER'); -- -- Data for Name: TestPlanRun; Type: TABLE DATA; Schema: public; Owner: atr @@ -133,6 +146,14 @@ INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults" INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (17, 1, 17, '[]'); INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (18, 1, 18, '[]'); INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (19, 1, 19, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (20, 1, 20, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (21, 2, 1, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (22, 2, 21, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (23, 2, 22, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (24, 2, 23, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (25, 2, 24, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (26, 2, 25, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (27, 2, 26, '[]'); -- -- Data for Name: CollectionJob; Type: TABLE DATA; Schema: public; Owner: atr diff --git a/server/scripts/populate-test-data/populateFakeTestResults.js b/server/scripts/populate-test-data/populateFakeTestResults.js index 28b7c2a94..dad1be2f2 100644 --- a/server/scripts/populate-test-data/populateFakeTestResults.js +++ b/server/scripts/populate-test-data/populateFakeTestResults.js @@ -5,31 +5,57 @@ const { } = require('../../models/services/BrowserService'); const { query, mutate } = require('../../tests/util/graphql-test-utilities'); +const FAKE_RESULT_CONFLICTS_OPTIONS = { + SINGLE: 1, + ALL: Infinity +}; + +/** + * + * @param {number} testPlanRunId + * @param {string['completeAndPassing' | + * 'completeAndFailingDueToIncorrectAssertions' | + * 'completeAndFailingDueToNoOutputAssertions' | + * 'completeAndFailingDueToUnexpectedBehaviors' | + * 'completeAndFailingDueToMultiple' | + * 'incompleteAndEmpty' | + * 'incompleteAndPassing' | + * 'incompleteAndFailingDueToIncorrectAssertions' | + * 'incompleteAndFailingDueToNoOutputAssertions' | + * 'incompleteAndFailingDueToUnexpectedBehaviors' | + * 'incompleteAndFailingDueToMultiple']} fakeTestResultTypes + * @param {import('sequelize').Transaction} transaction + * @param {number} numFakeTestResultConflicts + * @returns {Promise} + */ const populateFakeTestResults = async ( testPlanRunId, fakeTestResultTypes, - { transaction } + { + transaction, + numFakeTestResultConflicts = FAKE_RESULT_CONFLICTS_OPTIONS.SINGLE + } ) => { const { populateData: { testPlanReport } } = await query( gql` - query { - populateData(locationOfData: { testPlanRunId: ${testPlanRunId} }) { - testPlanReport { - at { - id - } - browser { - id - } - runnableTests { - id - } - } - } + query { + populateData(locationOfData: { testPlanRunId: ${testPlanRunId} }) { + testPlanReport { + at { + id + } + browser { + id + } + runnableTests { + id } - `, + } + } + } + `, { transaction } ); @@ -64,6 +90,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToIncorrectAssertions', submit: true, + numFakeTestResultConflicts, transaction }); break; @@ -74,6 +101,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToNoOutputAssertions', submit: true, + numFakeTestResultConflicts, transaction }); break; @@ -84,6 +112,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToUnexpectedBehaviors', submit: true, + numFakeTestResultConflicts, transaction }); break; @@ -94,6 +123,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToMultiple', submit: true, + numFakeTestResultConflicts, transaction }); break; @@ -124,6 +154,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToIncorrectAssertions', submit: false, + numFakeTestResultConflicts, transaction }); break; @@ -134,6 +165,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToNoOutputAssertions', submit: false, + numFakeTestResultConflicts, transaction }); break; @@ -144,6 +176,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToUnexpectedBehaviors', submit: false, + numFakeTestResultConflicts, transaction }); break; @@ -154,6 +187,7 @@ const populateFakeTestResults = async ( index, fakeTestResultType: 'failingDueToMultiple', submit: false, + numFakeTestResultConflicts, transaction }); break; @@ -190,6 +224,7 @@ const getFake = async ({ index, fakeTestResultType, submit, + numFakeTestResultConflicts, transaction }) => { const testId = testPlanReport.runnableTests[index].id; @@ -211,28 +246,28 @@ const getFake = async ({ } } = await mutate( gql` - mutation { - testPlanRun(id: ${testPlanRunId}) { - findOrCreateTestResult( - testId: "${testId}", - atVersionId: "${atVersion.id}", - browserVersionId: "${browserVersion.id}" - ) { - testResult { - id - scenarioResults { - id - output - assertionResults { - id - passed - } - } - } - } + mutation { + testPlanRun(id: ${testPlanRunId}) { + findOrCreateTestResult( + testId: "${testId}", + atVersionId: "${atVersion.id}", + browserVersionId: "${browserVersion.id}" + ) { + testResult { + id + scenarioResults { + id + output + assertionResults { + id + passed } + } } - `, + } + } + } + `, { transaction } ); @@ -255,60 +290,68 @@ const getFake = async ({ const testResult = getPassing(); - switch (fakeTestResultType) { - case 'empty': - return; - case 'passing': - break; - case 'failingDueToIncorrectAssertions': - testResult.scenarioResults[0].assertionResults[0].passed = false; - testResult.scenarioResults[0].assertionResults[0].failedReason = - 'INCORRECT_OUTPUT'; - break; - case 'failingDueToNoOutputAssertions': - testResult.scenarioResults[0].assertionResults[0].passed = false; - testResult.scenarioResults[0].assertionResults[0].failedReason = - 'NO_OUTPUT'; - break; - case 'failingDueToUnexpectedBehaviors': - testResult.scenarioResults[0].unexpectedBehaviors.push({ - id: 'OTHER', - impact: 'MODERATE', - details: 'Seeded other unexpected behavior' - }); - break; - case 'failingDueToMultiple': - testResult.scenarioResults[0].assertionResults[0].passed = false; - testResult.scenarioResults[0].assertionResults[0].failedReason = - 'INCORRECT_OUTPUT'; - testResult.scenarioResults[0].unexpectedBehaviors.push({ - id: 'EXCESSIVELY_VERBOSE', - impact: 'MODERATE', - details: 'N/A' - }); - testResult.scenarioResults[0].unexpectedBehaviors.push({ - id: 'OTHER', - impact: 'SEVERE', - details: 'Seeded other unexpected behavior' - }); - break; - default: - throw new Error(); + const applyResult = (scenarioResult, type) => { + switch (type) { + case 'failingDueToIncorrectAssertions': + scenarioResult.assertionResults[0].passed = false; + scenarioResult.assertionResults[0].failedReason = 'INCORRECT_OUTPUT'; + break; + case 'failingDueToNoOutputAssertions': + scenarioResult.assertionResults[0].passed = false; + scenarioResult.assertionResults[0].failedReason = 'NO_OUTPUT'; + break; + case 'failingDueToUnexpectedBehaviors': + scenarioResult.unexpectedBehaviors.push({ + id: 'OTHER', + impact: 'MODERATE', + details: 'Seeded other unexpected behavior' + }); + break; + case 'failingDueToMultiple': + scenarioResult.assertionResults[0].passed = false; + scenarioResult.assertionResults[0].failedReason = 'INCORRECT_OUTPUT'; + scenarioResult.unexpectedBehaviors.push( + { id: 'EXCESSIVELY_VERBOSE', impact: 'MODERATE', details: 'N/A' }, + { + id: 'OTHER', + impact: 'SEVERE', + details: 'Seeded other unexpected behavior' + } + ); + break; + default: + throw new Error(); + } + }; + + if (fakeTestResultType === 'empty') { + return; + } else if (fakeTestResultType === 'passing') { + // Do nothing + } else { + const setResult = scenarioResult => + applyResult(scenarioResult, fakeTestResultType); + + if (numFakeTestResultConflicts > 0) { + testResult.scenarioResults + .slice(0, numFakeTestResultConflicts) + .forEach(setResult); + } } const persistTestResult = submit ? 'submitTestResult' : 'saveTestResult'; await mutate( gql` - mutation PersistTestResult($input: TestResultInput!) { - testResult(id: "${testResult.id}") { - ${persistTestResult}(input: $input) { - locationOfData - } - } - } - `, + mutation PersistTestResult($input: TestResultInput!) { + testResult(id: "${testResult.id}") { + ${persistTestResult}(input: $input) { + locationOfData + } + } + } + `, { variables: { input: testResult }, transaction } ); }; -module.exports = populateFakeTestResults; +module.exports = { populateFakeTestResults, FAKE_RESULT_CONFLICTS_OPTIONS }; diff --git a/server/tests/integration/dataManagement.test.js b/server/tests/integration/dataManagement.test.js index e076903f9..6fe4c3920 100644 --- a/server/tests/integration/dataManagement.test.js +++ b/server/tests/integration/dataManagement.test.js @@ -34,75 +34,75 @@ const testPlanVersionsQuery = ({ }) => { return query( gql` - query { - testPlanVersions( - directory: ${directory} - phases: ${phases} - ) { + query { + testPlanVersions( + directory: ${directory} + phases: ${phases} + ) { + id + phase + gitSha + metadata + testPlan { + directory + } + testPlanReports { + id + markedFinalAt + at { + id + } + browser { + id + } + draftTestPlanRuns { + testResults { + id + completedAt + test { + id + rowNumber + title + ats { + id + name + } + scenarios { id - phase - gitSha - metadata - testPlan { - directory + commands { + id + text } - testPlanReports { - id - markedFinalAt - at { - id - } - browser { - id - } - draftTestPlanRuns { - testResults { - id - completedAt - test { - id - rowNumber - title - ats { - id - name - } - scenarios { - id - commands { - id - text - } - } - assertions { - id - priority - text - } - } - scenarioResults { - output - assertionResults { - id - assertion { - text - } - passed - } - scenario { - id - commands { - id - text - } - } - } - } - } + } + assertions { + id + priority + text + } + } + scenarioResults { + output + assertionResults { + id + assertion { + text } + passed + } + scenario { + id + commands { + id + text + } + } } + } } - `, + } + } + } + `, { transaction } ); }; @@ -114,87 +114,87 @@ const updateVersionToPhaseQuery = ( ) => { return mutate( gql` - mutation { - testPlanVersion(id: ${testPlanVersionId}) { - updatePhase( - phase: ${phase} - testPlanVersionDataToIncludeId: ${ - testPlanVersionDataToIncludeId ?? null + mutation { + testPlanVersion(id: ${testPlanVersionId}) { + updatePhase( + phase: ${phase} + testPlanVersionDataToIncludeId: ${ + testPlanVersionDataToIncludeId ?? null + } + ) { + testPlanVersion { + id + phase + gitSha + metadata + testPlan { + directory + } + testPlanReports { + id + markedFinalAt + at { + id + } + minimumAtVersion { + id + } + exactAtVersion { + id + } + browser { + id + } + draftTestPlanRuns { + testResults { + id + completedAt + test { + id + rowNumber + title + ats { + id + name + } + scenarios { + id + commands { + id + text } - ) { - testPlanVersion { - id - phase - gitSha - metadata - testPlan { - directory - } - testPlanReports { - id - markedFinalAt - at { - id - } - minimumAtVersion { - id - } - exactAtVersion { - id - } - browser { - id - } - draftTestPlanRuns { - testResults { - id - completedAt - test { - id - rowNumber - title - ats { - id - name - } - scenarios { - id - commands { - id - text - } - } - assertions { - id - priority - text - } - } - scenarioResults { - output - assertionResults { - id - assertion { - text - } - passed - } - scenario { - id - commands { - id - text - } - } - } - } - } - } + } + assertions { + id + priority + text + } + } + scenarioResults { + output + assertionResults { + id + assertion { + text } + passed + } + scenario { + id + commands { + id + text + } + } } + } } + } } - `, + } + } + } + `, { transaction } ); }; @@ -208,32 +208,32 @@ const createTestPlanReportQuery = ( ) => { return mutate( gql` - mutation { - createTestPlanReport(input: { - testPlanVersionId: ${testPlanVersionId} - atId: ${atId} - minimumAtVersionId: ${minimumAtVersionId} - browserId: ${browserId} - }) { - testPlanReport { - id - at { - id - } - browser { - id - } - } - testPlanVersion { - id - phase - testPlanReports { - id - } - } - } + mutation { + createTestPlanReport(input: { + testPlanVersionId: ${testPlanVersionId} + atId: ${atId} + minimumAtVersionId: ${minimumAtVersionId} + browserId: ${browserId} + }) { + testPlanReport { + id + at { + id } - `, + browser { + id + } + } + testPlanVersion { + id + phase + testPlanReports { + id + } + } + } + } + `, { transaction } ); }; @@ -241,17 +241,17 @@ const createTestPlanReportQuery = ( const markAsFinal = (testPlanReportId, { transaction }) => { return mutate( gql` - mutation { - testPlanReport(id: ${testPlanReportId}) { - markAsFinal { - testPlanReport { - id - markedFinalAt - } - } - } + mutation { + testPlanReport(id: ${testPlanReportId}) { + markAsFinal { + testPlanReport { + id + markedFinalAt } - `, + } + } + } + `, { transaction } ); }; @@ -306,8 +306,12 @@ const getTestableTestPlanVersions = async ({ // remove any newer version that deprecated it await rawQuery( ` - delete from "TestPlanReport" where "testPlanVersionId" = ${newTestPlanVersion.id}; - delete from "TestPlanVersion" where id = ${newTestPlanVersion.id};`, + delete + from "TestPlanReport" + where "testPlanVersionId" = ${newTestPlanVersion.id}; + delete + from "TestPlanVersion" + where id = ${newTestPlanVersion.id};`, { transaction } ); @@ -405,15 +409,15 @@ describe('data management', () => { const previous = await query( gql` - query { - testPlanVersion(id: ${testPlanVersionId}) { - phase - testPlanReports { - id - } - } - } - `, + query { + testPlanVersion(id: ${testPlanVersionId}) { + phase + testPlanReports { + id + } + } + } + `, { transaction } ); const previousPhase = previous.testPlanVersion.phase; @@ -435,12 +439,12 @@ describe('data management', () => { const testPlanReportsToMarkAsFinalResult = await query( gql` - query { - testPlanReports(testPlanVersionId: ${testPlanVersionId}) { - id - } - } - `, + query { + testPlanReports(testPlanVersionId: ${testPlanVersionId}) { + id + } + } + `, { transaction } ); diff --git a/server/tests/integration/testQueue.test.js b/server/tests/integration/testQueue.test.js index b74b9062e..42c598471 100644 --- a/server/tests/integration/testQueue.test.js +++ b/server/tests/integration/testQueue.test.js @@ -85,19 +85,19 @@ describe('test queue', () => { await dbCleaner(async transaction => { // A1 const testReportId = '1'; - const newTesterId = '2'; + const newTesterId = '3'; const previous = await query( gql` - query { - testPlanReport(id: ${testReportId}) { - draftTestPlanRuns { - tester { - id - } - } - } - } - `, + query { + testPlanReport(id: ${testReportId}) { + draftTestPlanRuns { + tester { + id + } + } + } + } + `, { transaction } ); const previousTesterIds = previous.testPlanReport.draftTestPlanRuns.map( @@ -107,30 +107,30 @@ describe('test queue', () => { // A2 const result = await mutate( gql` - mutation { - testPlanReport(id: ${testReportId}) { - assignTester(userId: ${newTesterId}) { - testPlanReport { - draftTestPlanRuns { - tester { - id - username - } - } - } - } - } + mutation { + testPlanReport(id: ${testReportId}) { + assignTester(userId: ${newTesterId}) { + testPlanReport { + draftTestPlanRuns { + tester { + id + username } - `, + } + } + } + } + } + `, { transaction } ); // prettier-ignore const resultTesterIds = result - .testPlanReport - .assignTester - .testPlanReport - .draftTestPlanRuns.map(testPlanRun => testPlanRun.tester.id); + .testPlanReport + .assignTester + .testPlanReport + .draftTestPlanRuns.map(testPlanRun => testPlanRun.tester.id); // A3 expect(previousTesterIds).not.toEqual( @@ -147,16 +147,16 @@ describe('test queue', () => { const previousTesterId = '1'; const previous = await query( gql` - query { - testPlanReport(id: ${testPlanReportId}) { - draftTestPlanRuns { - tester { - id - } - } - } - } - `, + query { + testPlanReport(id: ${testPlanReportId}) { + draftTestPlanRuns { + tester { + id + } + } + } + } + `, { transaction } ); const previousTesterIds = previous.testPlanReport.draftTestPlanRuns.map( @@ -166,29 +166,29 @@ describe('test queue', () => { // A2 const result = await mutate( gql` - mutation { - testPlanReport(id: ${testPlanReportId}) { - deleteTestPlanRun(userId: ${previousTesterId}) { - testPlanReport { - draftTestPlanRuns { - tester { - username - } - } - } - } - } + mutation { + testPlanReport(id: ${testPlanReportId}) { + deleteTestPlanRun(userId: ${previousTesterId}) { + testPlanReport { + draftTestPlanRuns { + tester { + username } - `, + } + } + } + } + } + `, { transaction } ); // prettier-ignore const resultTesterIds = result - .testPlanReport - .deleteTestPlanRun - .testPlanReport - .draftTestPlanRuns.map((run) => run.tester.id); + .testPlanReport + .deleteTestPlanRun + .testPlanReport + .draftTestPlanRuns.map((run) => run.tester.id); // A3 expect(previousTesterIds).toEqual( @@ -280,29 +280,29 @@ describe('test queue', () => { // A2 const result = await mutate( gql` - mutation { - createTestPlanReport(input: { - testPlanVersionId: ${testPlanVersionId} - atId: ${atId} - minimumAtVersionId: ${minimumAtVersionId} - browserId: ${browserId} - }) { - testPlanReport { - id - at { - id - } - browser { - id - } - } - testPlanVersion { - id - phase - } - } - } - `, + mutation { + createTestPlanReport(input: { + testPlanVersionId: ${testPlanVersionId} + atId: ${atId} + minimumAtVersionId: ${minimumAtVersionId} + browserId: ${browserId} + }) { + testPlanReport { + id + at { + id + } + browser { + id + } + } + testPlanVersion { + id + phase + } + } + } + `, { transaction } ); const { testPlanReport, testPlanVersion } = result.createTestPlanReport; @@ -333,39 +333,39 @@ describe('test queue', () => { const testPlanReportId = '4'; const queryBefore = await query( gql` - query { - testPlanReport(id: ${testPlanReportId}) { - id - draftTestPlanRuns { - id - } - } - } - `, + query { + testPlanReport(id: ${testPlanReportId}) { + id + draftTestPlanRuns { + id + } + } + } + `, { transaction } ); const { draftTestPlanRuns } = queryBefore.testPlanReport; await mutate( gql` - mutation { - testPlanReport(id: ${testPlanReportId}) { - deleteTestPlanReport - } - } - `, + mutation { + testPlanReport(id: ${testPlanReportId}) { + deleteTestPlanReport + } + } + `, { transaction } ); const queryAfter = await query( gql` - query { - testPlanReport(id: ${testPlanReportId}) { - id - } - testPlanRun(id: ${draftTestPlanRuns[0].id}) { - id - } - } - `, + query { + testPlanReport(id: ${testPlanReportId}) { + id + } + testPlanRun(id: ${draftTestPlanRuns[0].id}) { + id + } + } + `, { transaction } ); @@ -381,45 +381,45 @@ describe('test queue', () => { const result = await query( gql` - query { - testPlanReport(id: ${conflictingReportId}) { - conflicts { - source { - test { - title - rowNumber - } - scenario { - commands { - text - } - } - assertion { - text - } - } - conflictingResults { - testPlanRun { - id - tester { - username - } - } - scenarioResult { - output - unexpectedBehaviors { - text - details - } - } - assertionResult { - passed - } - } - } - } + query { + testPlanReport(id: ${conflictingReportId}) { + conflicts { + source { + test { + title + rowNumber + } + scenario { + commands { + text + } + } + assertion { + text + } + } + conflictingResults { + testPlanRun { + id + tester { + username + } } - `, + scenarioResult { + output + unexpectedBehaviors { + text + details + } + } + assertionResult { + passed + } + } + } + } + } + `, { transaction: false } ); diff --git a/server/tests/models/services/TestPlanRunService.test.js b/server/tests/models/services/TestPlanRunService.test.js index 92a85ae56..8cc6fc462 100644 --- a/server/tests/models/services/TestPlanRunService.test.js +++ b/server/tests/models/services/TestPlanRunService.test.js @@ -103,7 +103,7 @@ describe('TestPlanRunModel Data Checks', () => { it('should create testPlanRun if none exists for tester', async () => { await dbCleaner(async transaction => { const _testPlanReportId = 1; - const _testerUserId = 2; + const _testerUserId = 3; const testPlanRuns = await TestPlanRunService.getTestPlanRuns({ transaction