Test using Maestro CLI + emulator instead of Cloud (#4092)

Add Maestro local CI workflow:
- Remove previous Maestro Cloud.
- Use an emulator with Pixel 7 Pro - API 35.
- Allow to record several videos in the background to verify the run.
- Upload test results.
- Allow either dispatching a new flow, running the 'build apk' job or run with a PR after the 'Build APK' flow has succeeded.

.github/workflows/build.yml

@@ -54,6 +54,16 @@ jobs:
           path: |
             app/build/outputs/apk/gplay/debug/*-universal-debug.apk
             app/build/outputs/apk/fdroid/debug/*-universal-debug.apk
+      - name: Upload x86_64 APK for Maestro
+        if: ${{ matrix.variant == 'debug' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: elementx-apk-maestro
+          path: |
+            app/build/outputs/apk/gplay/debug/app-gplay-x86_64-debug.apk
+          retention-days: 5
+          overwrite: true
+          if-no-files-found: error
       - uses: rnkdsh/action-upload-diawi@v1.5.5
         id: diawi
         # Do not fail the whole build if Diawi upload fails

.github/workflows/maestro-local.yml

@@ -1,40 +1,39 @@
-name: Maestro
+name: Maestro (local)
 
-# Run this flow only on pull request, and only when the pull request has the Run-Maestro label, to limit our usage of maestro cloud.
+# Run this flow only when APK Build workflow completes
 on:
   workflow_dispatch:
-  pull_request:
+  workflow_run:
-    types: [labeled]
+    workflows: [APK Build]
+    types:
+      - completed
 
 # Enrich gradle.properties for CI/CD
 env:
   GRADLE_OPTS: -Dorg.gradle.jvmargs=-Xmx9g -XX:MaxMetaspaceSize=512m -Dfile.encoding=UTF-8 -XX:+HeapDumpOnOutOfMemoryError -XX:+UseG1GC -Dkotlin.daemon.jvm.options=-Xmx4g
   CI_GRADLE_ARG_PROPERTIES: --stacktrace --no-daemon -Dsonar.gradle.skipCompile=true --no-configuration-cache
+  ARCH: x86_64
+  DEVICE: pixel_7_pro
+  API_LEVEL: 35
+  TARGET: google_apis
 
 jobs:
   build-apk:
     name: Build APK
     runs-on: ubuntu-latest
-    if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'Run-Maestro'
+    if: github.event_name == 'workflow_dispatch'
     # Allow one per PR.
     concurrency:
       group: ${{ format('maestro-{0}', github.ref) }}
       cancel-in-progress: true
     steps:
-      - name: Remove Run-Maestro label
-        if: ${{ github.event_name == 'pull_request' && github.event.label.name == 'Run-Maestro' }}
-        uses: actions-ecosystem/action-remove-labels@v1
-        with:
-          labels: Run-Maestro
       - uses: actions/checkout@v4
-        if: (github.event_name == 'pull_request' && github.event.pull_request.fork == null) || github.event_name == 'workflow_dispatch'
         with:
           # Ensure we are building the branch and not the branch after being merged on develop
           # https://github.com/actions/checkout/issues/881
-          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.ref }}
+          ref: ${{ github.ref }}
       - uses: actions/setup-java@v4
         name: Use JDK 21
-        if: (github.event_name == 'pull_request' && github.event.pull_request.fork == null) || github.event_name == 'workflow_dispatch'
         with:
           distribution: 'temurin' # See 'Supported distributions' for available options
           java-version: '21'
@@ -44,7 +43,6 @@ jobs:
           cache-read-only: ${{ github.ref != 'refs/heads/develop' }}
       - name: Assemble debug APK
         run: ./gradlew :app:assembleGplayDebug $CI_GRADLE_ARG_PROPERTIES
-        if: (github.event_name == 'pull_request' && github.event.pull_request.fork == null) || github.event_name == 'workflow_dispatch'
         env:
           ELEMENT_ANDROID_MAPTILER_API_KEY: ${{ secrets.MAPTILER_KEY }}
           ELEMENT_ANDROID_MAPTILER_LIGHT_MAP_ID: ${{ secrets.MAPTILER_LIGHT_MAP_ID }}
@@ -62,8 +60,9 @@ jobs:
   maestro-cloud:
     name: Maestro test suite
     runs-on: ubuntu-latest
-    needs: build-apk
+    needs: [build-apk]
-    if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'Run-Maestro'
+    # Only if the APKs were built successfully.
+    if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success'
     # Allow one per PR.
     concurrency:
       group: ${{ format('maestro-{0}', github.ref) }}
@@ -74,23 +73,52 @@ jobs:
         with:
           # Ensure we are building the branch and not the branch after being merged on develop
           # https://github.com/actions/checkout/issues/881
-          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.ref }}
+          ref: ${{ github.ref }}
-      - name: Download APK artifact from previous job
+      - name: Download APK artifact from 'Build APKs' flow
         uses: actions/download-artifact@v4
+        if: github.event.workflow_run.conclusion == 'success'
         with:
           name: elementx-apk-maestro
-      - uses: mobile-dev-inc/action-maestro-cloud@v1.9.7
+          run-id: ${{ github.event.workflow_run.id }}
-        if: (github.event_name == 'pull_request' && github.event.pull_request.fork == null) || github.event_name == 'workflow_dispatch'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Download APK artifact from previous job
+        uses: actions/download-artifact@v4
+        if: github.event_name == 'workflow_dispatch'
         with:
-          api-key: ${{ secrets.MAESTRO_CLOUD_API_KEY }}
+          name: elementx-apk-maestro
-          # Doc says (https://github.com/mobile-dev-inc/action-maestro-cloud#android):
+      - name: Enable KVM group perms
-          # app-file should point to an x86 compatible APK file, so upload the x86_64 one (much smaller than the universal APK).
+        run: |
-          app-file: app-gplay-x86_64-debug.apk
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
-          env: |
+          sudo udevadm control --reload-rules
-            MAESTRO_USERNAME=maestroelement
+          sudo udevadm trigger --name-match=kvm
-            MAESTRO_PASSWORD=${{ secrets.MATRIX_MAESTRO_ACCOUNT_PASSWORD }}
+      - name: Install maestro
-            MAESTRO_RECOVERY_KEY=${{ secrets.MATRIX_MAESTRO_ACCOUNT_RECOVERY_KEY }}
+        run: curl -fsSL "https://get.maestro.mobile.dev" | bash
-            MAESTRO_ROOM_NAME=MyRoom
+      - name: Run Maestro tests in emulator
-            MAESTRO_INVITEE1_MXID=@maestroelement2:matrix.org
+        uses: reactivecircus/android-emulator-runner@v2
-            MAESTRO_INVITEE2_MXID=@maestroelement3:matrix.org
+        env:
-            MAESTRO_APP_ID=io.element.android.x.debug
+          MAESTRO_USERNAME: maestroelement
+          MAESTRO_PASSWORD: ${{ secrets.MATRIX_MAESTRO_ACCOUNT_PASSWORD }}
+          MAESTRO_RECOVERY_KEY: ${{ secrets.MATRIX_MAESTRO_ACCOUNT_RECOVERY_KEY }}
+          MAESTRO_ROOM_NAME: MyRoom
+          MAESTRO_INVITEE1_MXID: "@maestroelement2:matrix.org"
+          MAESTRO_INVITEE2_MXID: "@maestroelement3:matrix.org"
+          MAESTRO_APP_ID: io.element.android.x.debug
+        with:
+          api-level: ${{ env.API_LEVEL }}
+          arch: ${{ env.ARCH }}
+          profile: ${{ env.DEVICE }}
+          target: ${{ env.TARGET }}
+          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          disable-animations: true
+          disk-size: 3G
+          script: |
+            .github/workflows/scripts/maestro/maestro-local-with-screen-recording.sh app-gplay-x86_64-debug.apk
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results
+          path: |
+            ~/.maestro/tests/**
+          retention-days: 5
+          overwrite: true
+          if-no-files-found: error

.github/workflows/scripts/maestro/local-recording.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+#
+# Copyright 2024 New Vector Ltd.
+#
+# SPDX-License-Identifier: AGPL-3.0-only
+# Please see LICENSE in the repository root for full details.
+#
+
+COUNT=0
+mkdir -p /data/local/tmp/recordings;
+FILENAME=/data/local/tmp/recordings/testRecording$COUNT.mp4
+while true
+ do
+   ((COUNT++))
+   FILENAME=/data/local/tmp/recordings/testRecording$COUNT.mp4
+   echo "\nRecording video file #$COUNT"
+   screenrecord --bugreport --bit-rate=16m --size 720x1280 $FILENAME
+ done

.github/workflows/scripts/maestro/maestro-local-with-screen-recording.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+#
+# Copyright 2024 New Vector Ltd.
+#
+# SPDX-License-Identifier: AGPL-3.0-only
+# Please see LICENSE in the repository root for full details.
+#
+
+adb install -r $1
+set -x
+echo "Starting the screen recording..."
+adb push .github/workflows/scripts/maestro/local-recording.sh /data/local/tmp/
+adb shell "chmod +x /data/local/tmp/local-recording.sh"
+adb shell "/data/local/tmp/local-recording.sh & echo \$! > /data/local/tmp/screenrecord_pid.txt" &
+set +e
+~/.maestro/bin/maestro test .maestro/allTests.yaml
+TEST_STATUS=$?
+echo "Test run completed with status $TEST_STATUS"
+
+# Stop the screen recording loop
+SCRIPT_PID=$(adb shell "cat /data/local/tmp/screenrecord_pid.txt")
+adb shell "kill -2 $SCRIPT_PID"
+
+# Get the PID of the screen recording process
+SCREENRECORD_PID=$(adb shell ps | grep screenrecord | awk '{print $2}')
+# Wait for the screen recording process to exit
+while [ ! -z $SCREENRECORD_PID ]; do
+  echo "Waiting for screen recording ($SCREENRECORD_PID) to finish..."
+  adb shell "kill -2 $SCREENRECORD_PID"
+  sleep 1
+  SCREENRECORD_PID=$(adb shell ps | grep screenrecord | awk '{print $2}')
+done
+
+adb pull /data/local/tmp/recordings/ ~/.maestro/tests/
+exit $TEST_STATUS

.maestro/tests/account/changeServer.yaml

@@ -11,6 +11,9 @@ appId: ${MAESTRO_APP_ID}
     id: "change_server-server"
 - inputText: "element"
 - hideKeyboard
+- extendedWaitUntil:
+    visible: "element.io"
+    timeout: 10000
 - tapOn: "element.io"
 # Revert to matrix.org
 - tapOn:

.maestro/tests/roomList/createAndDeleteRoom.yaml

@@ -26,6 +26,10 @@ appId: ${MAESTRO_APP_ID}
 - tapOn: "Invite"
 - tapOn: "Back"
 - tapOn: "aRoomName"
+- scrollUntilVisible:
+    direction: DOWN
+    element:
+      text: "People"
 - tapOn: "People"
 # assert there's 1 member and 2 invitees
 - tapOn: "Back"