From 2a43d7403cfc5eefb3101da4d5d044a4d3d54152 Mon Sep 17 00:00:00 2001 From: Alexander Sosedkin Date: Thu, 13 Jun 2024 20:54:52 +0200 Subject: [PATCH] fixup emulator-based tests --- .github/workflows/emulator.yml | 43 +++++++++---------- ...strap-channel.py => bootstrap_channels.py} | 16 +++++-- ...ootstrap-flakes.py => bootstrap_flakes.py} | 13 +++--- tests/emulator/common.py | 15 ++++++- tests/emulator/on-device-tests.py | 23 ---------- tests/emulator/on_device_tests.py | 27 ++++++++++++ tests/emulator/test.py | 13 ++++++ tests/emulator/test_channels_shell.py | 37 ++++++++++++++++ tests/emulator/test_channels_uiautomator.py | 7 +++ 9 files changed, 137 insertions(+), 57 deletions(-) rename tests/emulator/{bootstrap-channel.py => bootstrap_channels.py} (69%) rename tests/emulator/{bootstrap-flakes.py => bootstrap_flakes.py} (70%) delete mode 100644 tests/emulator/on-device-tests.py create mode 100644 tests/emulator/on_device_tests.py create mode 100644 tests/emulator/test.py create mode 100644 tests/emulator/test_channels_shell.py create mode 100644 tests/emulator/test_channels_uiautomator.py diff --git a/.github/workflows/emulator.yml b/.github/workflows/emulator.yml index 2c1ee516..9623c6ae 100644 --- a/.github/workflows/emulator.yml +++ b/.github/workflows/emulator.yml @@ -6,13 +6,14 @@ on: - cron: 0 0 * * 1 jobs: - emulator: + emulate: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - api-level: [29] - way: [flakes, channel] + api-level: [29, 34] + way: [bootstrap_flakes, test_channels_uiautomator, test_channels_shell] steps: - name: Checkout repository @@ -27,6 +28,9 @@ jobs: name: nix-on-droid signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}" + - name: Build droidctl + run: nix build 'github:t184256/droidctl' --out-link droidctl + - name: Configure AVD cache uses: actions/cache@v4 id: avd-cache @@ -40,42 +44,37 @@ jobs: if: steps.avd-cache.outputs.cache-hit != 'true' uses: reactivecircus/android-emulator-runner@v2 with: + target: default api-level: ${{ matrix.api-level }} + arch: x86_64 force-avd-creation: false emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none - disable-animations: false + disable-animations: true script: echo "Generated AVD snapshot for caching." - - name: Test using channels api-level=${{ matrix.api-level }} - if: ${{ matrix.way == 'channel' }} + - name: Test way=${{ matrix.way}} api-level=${{ matrix.api-level }} uses: reactivecircus/android-emulator-runner@v2 with: target: default api-level: ${{ matrix.api-level }} arch: x86_64 - script: | - pushd tests/emulator - nix run . -- run bootstrap-channel.py - nix run . -- run on-device-tests.py - popd + script: > + cd tests/emulator && + adb shell settings put secure enabled_accessibility_services com.google.android.marvin.talkback/com.google.android.marvin.talkback.TalkBackService && + nix run 'github:t184256/droidctl' -- run ${{ matrix.way }}.py - - name: Bootstrap using flakes api-level=${{ matrix.api-level }} - if: ${{ matrix.way == 'flakes' }} - uses: reactivecircus/android-emulator-runner@v2 + - name: Upload screenshots + if: always() + uses: actions/upload-artifact@v4 with: - target: default - api-level: ${{ matrix.api-level }} - arch: x86_64 - script: | - pushd tests/emulator - nix run . -- run bootstrap-flakes.py - popd + name: screenshots-${{ matrix.way }}-${{ matrix.api-level }} + path: tests/emulator/screenshots + if-no-files-found: warn # 'error' or 'ignore' are also available # TODO: push to cachix from device # - name: Push to cachix # if: always() && github.event_name != 'pull_request' # run: ... -# TODO: parametrize: channel vs flakes # TODO: externally-driven test # TODO: more API levels diff --git a/tests/emulator/bootstrap-channel.py b/tests/emulator/bootstrap_channels.py similarity index 69% rename from tests/emulator/bootstrap-channel.py rename to tests/emulator/bootstrap_channels.py index 91502e02..bd52221f 100644 --- a/tests/emulator/bootstrap-channel.py +++ b/tests/emulator/bootstrap_channels.py @@ -1,4 +1,4 @@ -from common import wait_for, APK, BOOTSTRAP_URL +from common import screenshot, wait_for, APK, BOOTSTRAP_URL def run(d): @@ -10,6 +10,12 @@ def run(d): d.ui(className='android.widget.EditText').set_text(BOOTSTRAP_URL) d.ui(text='OK').click() + import time; time.sleep(30) + tv = d.ui(resourceId='com.termux.nix:id/terminal_view') + print(tv) + print(tv.info) + print(tv.info['contentDescription']) + wait_for(d, 'Welcome to Nix-on-Droid!') wait_for(d, 'Do you want to set it up with flakes? (y/N)') d.ui.press('enter') @@ -17,12 +23,14 @@ def run(d): wait_for(d, 'Installing and updating nix-channels...') wait_for(d, 'unpacking channels...') - wait_for(d, 'Installing first Nix-on-Droid generation...', timeout=180) + wait_for(d, 'Installing first Nix-on-Droid generation...', timeout=600) wait_for(d, 'Copying default Nix-on-Droid config...', timeout=180) wait_for(d, 'Congratulations!') - wait_for(d, 'See config file for further information!') - wait_for(d, 'bash-5.25$') + wait_for(d, 'See config file for further information.') + wait_for(d, 'bash-5.2$') d('input text "echo smoke-test | base64"') # remove d.ui.press('enter') wait_for(d, 'c21va2UtdGVzdAo=') + + screenshot(d, 'success-bootstrap-channels') diff --git a/tests/emulator/bootstrap-flakes.py b/tests/emulator/bootstrap_flakes.py similarity index 70% rename from tests/emulator/bootstrap-flakes.py rename to tests/emulator/bootstrap_flakes.py index 24336f85..5d9127dc 100644 --- a/tests/emulator/bootstrap-flakes.py +++ b/tests/emulator/bootstrap_flakes.py @@ -12,16 +12,17 @@ def run(d): wait_for(d, 'Welcome to Nix-on-Droid!') wait_for(d, 'Do you want to set it up with flakes? (y/N)') - d('input text y') # remove + d('input text y') d.ui.press('enter') wait_for(d, 'Setting up Nix-on-Droid with flakes...') - ... - wait_for(d, 'Installing first Nix-on-Droid generation...', timeout=180) - wait_for(d, 'Copying default Nix-on-Droid config...', timeout=180) + wait_for(d, 'Installing flake from default template...') + wait_for(d, 'Overriding input urls / arch in flake...') + wait_for(d, 'Installing first Nix-on-Droid generation...', timeout=600) + wait_for(d, 'Building activation package', timeout=900) wait_for(d, 'Congratulations!') - wait_for(d, 'See config file for further information!') - wait_for(d, 'bash-5.25$') + wait_for(d, 'See config file for further information.') + wait_for(d, 'bash-5.2$') d('input text "echo smoke-test | base64"') # remove d.ui.press('enter') diff --git a/tests/emulator/common.py b/tests/emulator/common.py index 749ae972..07fffd73 100644 --- a/tests/emulator/common.py +++ b/tests/emulator/common.py @@ -1,12 +1,22 @@ +import os import sys import time SERVER = 'https://nix-on-droid.unboiled.info' APK = f'{SERVER}/com.termux.nix_188035-x86_64.apk' -BOOTSTRAP_URL = f'{SERVER}/testing' +BOOTSTRAP_URL = f'{SERVER}/bootstrap-testing' -def wait_for(d, on_screen_text, timeout=30, critical=True): +def screenshot(d, suffix=''): + os.makedirs('screenshots', exist_ok=True) + fname_base = f'screenshots/{time.time()}-{suffix}' + d.ui.screenshot(f'{fname_base}.png') + with open(f'{fname_base}.xml', 'w') as f: + f.write(d.ui.dump_hierarchy()) + print(f'screenshotted: {fname_base}.{{png,xml}}') + + +def wait_for(d, on_screen_text, timeout=90, critical=True): start = time.time() last_displayed_time = None while (elapsed := time.time() - start) < timeout: @@ -19,5 +29,6 @@ def wait_for(d, on_screen_text, timeout=30, critical=True): return time.sleep(.75) print(f'NOT FOUND: {on_screen_text} after {timeout}s') + screenshot(d, suffix='error') if critical: sys.exit(1) diff --git a/tests/emulator/on-device-tests.py b/tests/emulator/on-device-tests.py deleted file mode 100644 index 4d6efbd6..00000000 --- a/tests/emulator/on-device-tests.py +++ /dev/null @@ -1,23 +0,0 @@ -from common import wait_for - - -def run(d): - #wait_for(d, 'bash-5.2$') - - #d('input text "nix-on-droid on-device-test"') - #wait_for(d, 'These semi-automated tests are destructive', timeout=180) - #wait_for(d, 'Proceeding will wreck your installation.') - #wait_for(d, 'Do you still wish to proceed?') - #d('input text "I do"') - #d.ui.press('enter') - - d.ui.open_notification() - d.ui(text='Nix').right(resourceId='android:id/expand_button').click() - d.ui(text='Acquire wakelock').click() - d.ui(text='Release wakelock').wait() - d.ui.press('back') - - if 'Allow' in d.ui.dump_hierarchy(): - d.ui.click('Allow') - - wait_for(d, 'tests, 0 failures in', timeout=1800) diff --git a/tests/emulator/on_device_tests.py b/tests/emulator/on_device_tests.py new file mode 100644 index 00000000..36fbc366 --- /dev/null +++ b/tests/emulator/on_device_tests.py @@ -0,0 +1,27 @@ +from common import screenshot, wait_for + + +def run(d): + wait_for(d, 'bash-5.2$') + + d('input text "nix-on-droid on-device-test"') + wait_for(d, 'These semi-automated tests are destructive', timeout=180) + wait_for(d, 'Proceeding will wreck your installation.') + wait_for(d, 'Do you still wish to proceed?') + d('input text "I do"') + d.ui.press('enter') + + d.ui.open_notification() + d.ui(text='Nix').right(resourceId='android:id/expand_button').click() + screenshot(d, 'notification_expanded') + d.ui(text='Acquire wakelock').click() + screenshot(d, 'wakelock_acquired') + d.ui(text='Release wakelock').wait() + screenshot(d, 'gotta go back') + d.ui.press('back') + screenshot(d, 'went back') + + if 'Allow' in d.ui.dump_hierarchy(): + d.ui.click('Allow') + + wait_for(d, 'tests, 0 failures in', timeout=1800) diff --git a/tests/emulator/test.py b/tests/emulator/test.py new file mode 100644 index 00000000..2a7ad5f3 --- /dev/null +++ b/tests/emulator/test.py @@ -0,0 +1,13 @@ +from common import screenshot, wait_for, APK, BOOTSTRAP_URL + + +def run(d): + nod = d.app('com.termux.nix', url=APK) + nod.permissions.allow_notifications() + nod.launch() + + tv = d.ui(resourceId='com.termux.nix:id/terminal_view') + print(tv) + print(tv.info) + print(tv.info['contentDescription']) + #wait_for(d, 'Welcome to Nix-on-Droid!') diff --git a/tests/emulator/test_channels_shell.py b/tests/emulator/test_channels_shell.py new file mode 100644 index 00000000..ddee3272 --- /dev/null +++ b/tests/emulator/test_channels_shell.py @@ -0,0 +1,37 @@ +import bootstrap_channels +import subprocess + +from common import wait_for + +TD = '/data/data/com.termux.nix/files/usr/home/.cache/nix-on-droid-self-test' + + +def run(d): + bootstrap_channels.run(d) + + d('input text "exit"') # remove + d.ui.press('enter') + + nod = d.app('com.termux.nix') + nod.launch() + + wait_for(d, 'bash-5.2$') + + # run tests in a way that'd display progress in CI + user = d.su('stat -c %U /data/data/com.termux.nix').output.strip() + # WARNING: assumes `su 0` style `su` that doesn't support -c from now on + print(f'{user=}') + for cmd in [ + 'id', + f'mkdir -p {TD}', + f'touch {TD}/confirmation-granted', + ('/data/data/com.termux.nix/files/usr/bin/login echo test'), + ('/data/data/com.termux.nix/files/usr/bin/login id'), + ('cd /data/data/com.termux.nix/files/home; ' + 'pwd; ' + 'id; ' + '/data/data/com.termux.nix/files/usr/bin/login ' + ' nix-on-droid on-device-test') + ]: + subprocess.run(['adb', 'shell', 'su', '0', 'su', user, + 'sh', '-c', cmd]) diff --git a/tests/emulator/test_channels_uiautomator.py b/tests/emulator/test_channels_uiautomator.py new file mode 100644 index 00000000..06be7c34 --- /dev/null +++ b/tests/emulator/test_channels_uiautomator.py @@ -0,0 +1,7 @@ +import bootstrap_channels +import on_device_tests + + +def run(d): + bootstrap_channels.run(d) + on_device_tests.run(d)