fixup emulator-based tests

.github/workflows/emulator.yml

@@ -6,13 +6,19 @@ on:
     - cron: 0 0 * * 1
 
 jobs:
-  emulator:
+  emulate:
     runs-on: ubuntu-latest
 
     strategy:
+      fail-fast: false
       matrix:
+        #api-level: [29, 34]
         api-level: [29]
-        way: [flakes, channel]
+        way:
+          - bootstrap_flakes
+          - bootstrap_channels
+          - test_channels_uiautomator
+          - test_channels_shell
 
     steps:
       - name: Checkout repository
@@ -27,6 +33,9 @@ jobs:
           name: nix-on-droid
           signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}"
 
+      - name: Build droidctl
+        run: nix build 'github:t184256/droidctl' --out-link droidctl
+
       - name: Configure AVD cache
         uses: actions/cache@v4
         id: avd-cache
@@ -40,42 +49,37 @@ jobs:
         if: steps.avd-cache.outputs.cache-hit != 'true'
         uses: reactivecircus/android-emulator-runner@v2
         with:
+          target: default
           api-level: ${{ matrix.api-level }}
+          arch: x86_64
           force-avd-creation: false
           emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
-          disable-animations: false
+          disable-animations: true
           script: echo "Generated AVD snapshot for caching."
 
-      - name: Test using channels api-level=${{ matrix.api-level }}
-        if: ${{ matrix.way == 'channel' }}
+      - name: Test way=${{ matrix.way}} api-level=${{ matrix.api-level }}
         uses: reactivecircus/android-emulator-runner@v2
         with:
           target: default
           api-level: ${{ matrix.api-level }}
           arch: x86_64
-          script: |
-            pushd tests/emulator
-            nix run . -- run bootstrap-channel.py
-            nix run . -- run on-device-tests.py
-            popd
+          script: >
+            cd tests/emulator &&
+            adb shell settings put secure enabled_accessibility_services com.google.android.marvin.talkback/com.google.android.marvin.talkback.TalkBackService &&
+            nix run 'github:t184256/droidctl' -- run ${{ matrix.way }}.py
 
-      - name: Bootstrap using flakes api-level=${{ matrix.api-level }}
-        if: ${{ matrix.way == 'flakes' }}
-        uses: reactivecircus/android-emulator-runner@v2
+      - name: Upload screenshots
+        if: always()
+        uses: actions/upload-artifact@v4
         with:
-          target: default
-          api-level: ${{ matrix.api-level }}
-          arch: x86_64
-          script: |
-            pushd tests/emulator
-            nix run . -- run bootstrap-flakes.py
-            popd
+          name: screenshots-${{ matrix.way }}-${{ matrix.api-level }}
+          path: tests/emulator/screenshots
+          if-no-files-found: warn # 'error' or 'ignore' are also available
 
 # TODO: push to cachix from device
 #     - name: Push to cachix
 #       if: always() && github.event_name != 'pull_request'
 #       run: ...
 
-# TODO: parametrize: channel vs flakes
 # TODO: externally-driven test
 # TODO: more API levels

tests/emulator/bootstrap-channel.py → tests/emulator/bootstrap_channels.py

@@ -1,4 +1,6 @@
-from common import wait_for, APK, BOOTSTRAP_URL
+from common import screenshot, wait_for, APK, BOOTSTRAP_URL
+
+import time
 
 
 def run(d):
@@ -8,7 +10,16 @@ def run(d):
 
     wait_for(d, 'Bootstrap zipball location')
     d.ui(className='android.widget.EditText').set_text(BOOTSTRAP_URL)
+    screenshot(d, 'entered-url')
+    for i in range(2):
+        if 'text="OK"' not in d.ui.dump_hierarchy():
+            d.ui.press('back')
+            time.sleep(.5)
+        else:
+            break
+    screenshot(d, 'entered-url-back')
     d.ui(text='OK').click()
+    screenshot(d, 'ok-clicked')
 
     wait_for(d, 'Welcome to Nix-on-Droid!')
     wait_for(d, 'Do you want to set it up with flakes? (y/N)')
@@ -17,12 +28,14 @@ def run(d):
 
     wait_for(d, 'Installing and updating nix-channels...')
     wait_for(d, 'unpacking channels...')
-    wait_for(d, 'Installing first Nix-on-Droid generation...', timeout=180)
+    wait_for(d, 'Installing first Nix-on-Droid generation...', timeout=600)
     wait_for(d, 'Copying default Nix-on-Droid config...', timeout=180)
     wait_for(d, 'Congratulations!')
-    wait_for(d, 'See config file for further information!')
-    wait_for(d, 'bash-5.25$')
+    wait_for(d, 'See config file for further information.')
+    wait_for(d, 'bash-5.2$')
 
     d('input text "echo smoke-test | base64"')  # remove
     d.ui.press('enter')
     wait_for(d, 'c21va2UtdGVzdAo=')
+
+    screenshot(d, 'success-bootstrap-channels')

tests/emulator/bootstrap_flakes.py

@@ -0,0 +1,41 @@
+from common import screenshot, wait_for, APK, BOOTSTRAP_URL
+
+import time
+
+
+def run(d):
+    nod = d.app('com.termux.nix', url=APK)
+    nod.permissions.allow_notifications()
+    nod.launch()
+
+    wait_for(d, 'Bootstrap zipball location')
+    d.ui(className='android.widget.EditText').set_text(BOOTSTRAP_URL)
+    screenshot(d, 'entered-url')
+    for i in range(2):
+        if 'text="OK"' not in d.ui.dump_hierarchy():
+            d.ui.press('back')
+            time.sleep(.5)
+        else:
+            break
+    screenshot(d, 'entered-url-back')
+    d.ui(text='OK').click()
+    screenshot(d, 'ok-clicked')
+
+    wait_for(d, 'Welcome to Nix-on-Droid!')
+    wait_for(d, 'Do you want to set it up with flakes? (y/N)')
+    d('input text y')
+    d.ui.press('enter')
+    wait_for(d, 'Setting up Nix-on-Droid with flakes...')
+
+    wait_for(d, 'Installing flake from default template...')
+    wait_for(d, 'Overriding input urls / arch in flake...')
+    wait_for(d, 'Installing first Nix-on-Droid generation...', timeout=600)
+    wait_for(d, 'Building activation package')
+    wait_for(d, 'Congratulations!', timeout=900)
+    wait_for(d, 'bash-5.2$')
+
+    d('input text "echo smoke-test | base64"')  # remove
+    d.ui.press('enter')
+    wait_for(d, 'c21va2UtdGVzdAo=')
+
+    screenshot(d, 'success-bootstrap-flakes')

tests/emulator/common.py

@@ -1,12 +1,22 @@
+import os
 import sys
 import time
 
 SERVER = 'https://nix-on-droid.unboiled.info'
 APK = f'{SERVER}/com.termux.nix_188035-x86_64.apk'
-BOOTSTRAP_URL = f'{SERVER}/testing'
+BOOTSTRAP_URL = f'{SERVER}/bootstrap-testing'
 
 
-def wait_for(d, on_screen_text, timeout=30, critical=True):
+def screenshot(d, suffix=''):
+    os.makedirs('screenshots', exist_ok=True)
+    fname_base = f'screenshots/{time.time()}-{suffix}'
+    d.ui.screenshot(f'{fname_base}.png')
+    with open(f'{fname_base}.xml', 'w') as f:
+        f.write(d.ui.dump_hierarchy())
+    print(f'screenshotted: {fname_base}.{{png,xml}}')
+
+
+def wait_for(d, on_screen_text, timeout=90, critical=True):
     start = time.time()
     last_displayed_time = None
     while (elapsed := time.time() - start) < timeout:
@@ -19,5 +29,6 @@ def wait_for(d, on_screen_text, timeout=30, critical=True):
             return
         time.sleep(.75)
     print(f'NOT FOUND: {on_screen_text} after {timeout}s')
+    screenshot(d, suffix='error')
     if critical:
         sys.exit(1)

tests/emulator/on_device_tests.py

@@ -0,0 +1,28 @@
+from common import screenshot, wait_for
+
+
+def run(d):
+    wait_for(d, 'bash-5.2$')
+
+    d('input text "nix-on-droid on-device-test"')
+    d.ui.press('enter')
+    wait_for(d, 'These semi-automated tests are destructive', timeout=180)
+    wait_for(d, 'Proceeding will wreck your installation.')
+    wait_for(d, 'Do you still wish to proceed?')
+    d('input text "I do"')
+    d.ui.press('enter')
+
+    d.ui.open_notification()
+    d.ui(text='Nix').right(resourceId='android:id/expand_button').click()
+    screenshot(d, 'notification_expanded')
+    d.ui(description='Acquire wakelock').click()
+    screenshot(d, 'wakelock_acquired')
+    d.ui(description='Release wakelock').wait()
+    screenshot(d, 'gotta go back')
+    d.ui.press('back')
+    screenshot(d, 'went back')
+
+    if 'Allow' in d.ui.dump_hierarchy():
+        d.ui(text='Allow').click()
+
+    wait_for(d, 'tests, 0 failures in', timeout=1800)

tests/emulator/test.py

@@ -0,0 +1,13 @@
+from common import screenshot, wait_for, APK, BOOTSTRAP_URL
+
+
+def run(d):
+    nod = d.app('com.termux.nix', url=APK)
+    nod.permissions.allow_notifications()
+    nod.launch()
+
+    tv = d.ui(resourceId='com.termux.nix:id/terminal_view')
+    print(tv)
+    print(tv.info)
+    print(tv.info['contentDescription'])
+    #wait_for(d, 'Welcome to Nix-on-Droid!')

tests/emulator/test_channels_shell.py

@@ -0,0 +1,37 @@
+import bootstrap_channels
+import subprocess
+
+from common import wait_for
+
+TD = '/data/data/com.termux.nix/files/usr/home/.cache/nix-on-droid-self-test'
+
+
+def run(d):
+    bootstrap_channels.run(d)
+
+    d('input text "exit"')  # remove
+    d.ui.press('enter')
+
+    nod = d.app('com.termux.nix')
+    nod.launch()
+
+    wait_for(d, 'bash-5.2$')
+
+    # run tests in a way that'd display progress in CI
+    user = d.su('stat -c %U /data/data/com.termux.nix').output.strip()
+    # WARNING: assumes `su 0` style `su` that doesn't support -c from now on
+    print(f'{user=}')
+    for cmd in [
+        'id',
+        f'mkdir -p {TD}',
+        f'touch {TD}/confirmation-granted',
+        ('/data/data/com.termux.nix/files/usr/bin/login echo test'),
+        ('/data/data/com.termux.nix/files/usr/bin/login id'),
+        ('cd /data/data/com.termux.nix/files/home; '
+         'pwd; '
+         'id; '
+         '/data/data/com.termux.nix/files/usr/bin/login '
+         ' nix-on-droid on-device-test')
+    ]:
+        subprocess.run(['adb', 'shell', 'su', '0', 'su', user,
+                        'sh', '-c', cmd])

tests/emulator/test_channels_uiautomator.py

@@ -0,0 +1,7 @@
+import bootstrap_channels
+import on_device_tests
+
+
+def run(d):
+    bootstrap_channels.run(d)
+    on_device_tests.run(d)