CI: make sure Paparazzi test failures stop the test flow (#117)

* Run Paparazzi tests just once when koverMergedReport is used.
* Make sure a Paparazzi failure breaks the test flow and blocks merging.
* Change ./gradlew check call in quality.yml to custom runQualityChecks task.

.github/workflows/quality.yml

@@ -1,6 +1,7 @@
 name: Code Quality Checks
 
 on:
+  workflow_dispatch:
   pull_request: { }
   push:
     branches: [ main, develop ]
@@ -21,7 +22,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Run code quality check suite
-        run: ./gradlew check $CI_GRADLE_ARG_PROPERTIES
+        run: ./gradlew runQualityChecks $CI_GRADLE_ARG_PROPERTIES
       - name: Upload reports
         if: always()
         uses: actions/upload-artifact@v3
@@ -29,15 +30,6 @@ jobs:
           name: linting-report
           path: |
             */build/reports/**/*.*
-      - name: Check Kover rules
-        run: ./gradlew koverMergedVerify $CI_GRADLE_ARG_PROPERTIES
-      - name: Upload reports
-        if: failure()
-        uses: actions/upload-artifact@v3
-        with:
-          name: kover-report
-          path: |
-            **/kover/merged/verification/errors.txt
       - name: Prepare Danger
         if: always()
         run: |

.github/workflows/tests.yml

@@ -1,6 +1,7 @@
 name: Test
 
 on:
+  workflow_dispatch:
   pull_request: { }
   push:
     branches: [ main, develop ]
@@ -8,7 +9,7 @@ on:
 # Enrich gradle.properties for CI/CD
 env:
   GRADLE_OPTS: -Dorg.gradle.jvmargs="-Xmx3072m -Dfile.encoding=UTF-8 -XX:+HeapDumpOnOutOfMemoryError" -Dkotlin.daemon.jvm.options="-Xmx2560m" -Dkotlin.incremental=false
-  CI_GRADLE_ARG_PROPERTIES: --stacktrace -PpreDexEnable=false --max-workers 4 --no-daemon
+  CI_GRADLE_ARG_PROPERTIES: --stacktrace -PpreDexEnable=false --max-workers 4
 
 jobs:
   tests:
@@ -20,11 +21,42 @@ jobs:
       group: ${{ github.ref == 'refs/heads/main' && format('unit-tests-main-{0}', github.sha) || github.ref == 'refs/heads/develop' && format('unit-tests-develop-{0}', github.sha) || format('unit-tests-{0}', github.ref) }}
       cancel-in-progress: true
     steps:
-      - uses: actions/checkout@v3
-      - name: Run tests and generate kover report
-        run: ./gradlew koverMergedReport $CI_GRADLE_ARG_PROPERTIES
+      - name: ⏬ Checkout with LFS
+        uses: actions/checkout@v3
+        with:
+          lfs: 'true'
 
-      - name: Archive kover report
+      - name: ⚙️ Run unit & screenshot tests, generate kover report
+        run: ./gradlew koverMergedReport $CI_GRADLE_ARG_PROPERTIES -Pci-build=true
+
+      - name: 📈 Verify coverage
+        run: ./gradlew koverMergedVerify $CI_GRADLE_ARG_PROPERTIES -Pci-build=true
+
+      - name: 🚫 Upload kover failed coverage reports
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: kover-error-report
+          path: |
+            **/kover/merged/verification/errors.txt
+
+      - name: 📸 Upload Screenshot test report
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: reports
+          path: tests/uitests/build/reports/tests/testDebugUnitTest/
+          retention-days: 5
+
+      - name: 🚫 Upload Screenshot failure differences on error
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: failures
+          path: tests/uitests/out/failures/
+          retention-days: 5
+
+      - name: ✅ Upload kover report
         if: always()
         uses: actions/upload-artifact@v3
         with:
@@ -32,7 +64,7 @@ jobs:
           path: |
             **/build/reports/kover/merged
 
-      - name: Archive test results on error
+      - name: 🚫 Upload test results on error
         if: failure()
         uses: actions/upload-artifact@v3
         with:
@@ -41,7 +73,7 @@ jobs:
             **/out/failures/
             **/build/reports/tests/*UnitTest/
 
-      - name: Publish results to Sonar
+      - name: 🔊 Publish results to Sonar
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
           ORG_GRADLE_PROJECT_SONAR_LOGIN: ${{ secrets.SONAR_TOKEN }}
@@ -49,7 +81,7 @@ jobs:
         run: ./gradlew sonar $CI_GRADLE_ARG_PROPERTIES
 
       # https://github.com/codecov/codecov-action
-      - name: Upload coverage reports to codecov
+      - name: ☂️ Upload coverage reports to codecov
         if: always()
         uses: codecov/codecov-action@v3
         # with:

build.gradle.kts

@@ -1,3 +1,6 @@
+import kotlinx.kover.api.KoverTaskExtension
+import org.jetbrains.kotlin.cli.common.toBooleanLenient
+
 /*
  * Copyright (c) 2022 New Vector Ltd
  *
@@ -205,11 +208,11 @@ koverMerged {
             name = "Global minimum code coverage."
             target = kotlinx.kover.api.VerificationTarget.ALL
             bound {
-                minValue = 55
+                minValue = 50
                 // Setting a max value, so that if coverage is bigger, it means that we have to change minValue.
                 // For instance if we have minValue = 25 and maxValue = 30, and current code coverage is now 37.32%, update
                 // minValue to 35 and maxValue to 40.
-                maxValue = 60
+                maxValue = 55
                 counter = kotlinx.kover.api.CounterType.INSTRUCTION
                 valueType = kotlinx.kover.api.VerificationValueType.COVERED_PERCENTAGE
             }
@@ -257,3 +260,40 @@ koverMerged {
         }
     }
 }
+
+// Make Kover depend on Paparazzi
+tasks.whenTaskAdded {
+    if (name.startsWith("koverMerged")) {
+        dependsOn(":tests:uitests:verifyPaparazziDebug")
+    }
+}
+
+// When running on the CI, run only debug test variants
+val ciBuildProperty = "ci-build"
+val isCiBuild = if (project.hasProperty(ciBuildProperty)) {
+    val raw = project.property(ciBuildProperty) as? String
+    raw?.toBooleanLenient() == true || raw?.toIntOrNull() == 1
+} else false
+if (isCiBuild) {
+    allprojects {
+        afterEvaluate {
+            tasks.withType<Test>().configureEach {
+                extensions.configure<KoverTaskExtension> {
+                    val enabled = name.contains("debug", ignoreCase = true)
+                    isDisabled.set(!enabled)
+                }
+            }
+        }
+    }
+}
+
+// Register quality check tasks.
+tasks.register("runQualityChecks") {
+    project.subprojects {
+        // For some reason `findByName("lint")` doesn't work
+        tasks.findByPath("$path:lint")?.let { dependsOn(it) }
+        tasks.findByName("detekt")?.let { dependsOn(it) }
+        tasks.findByName("ktlintCheck")?.let { dependsOn(it) }
+    }
+    dependsOn(":app:knitCheck")
+}

changelog.d/117.misc

@@ -0,0 +1,4 @@
+Changes to CI:
+- Run Paparazzi tests just once when `koverMergedReport` is used.
+- Make sure a Paparazzi failure breaks the test flow and blocks merging.
+- Change `./gradlew check` call in `quality.yml` to custom `runQualityChecks` task.

tests/uitests/src/test/kotlin/io/element/android/tests/uitests/ScreenshotTest.kt

@@ -67,7 +67,7 @@ class ScreenshotTest {
 
     @get:Rule
     val paparazzi = Paparazzi(
-        maxPercentDifference = 0.0,
+        maxPercentDifference = 0.01,
     )
 
     @Test

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline.components.virtual_null_DefaultGroup_TimelineItemDaySeparatorViewDarkPreview_0_null_0,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40e3f8efb419854d56f195fc904a9024a872d849b06e0a12ae23f1aca5c39e14
+size 6246

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline.components.virtual_null_DefaultGroup_TimelineItemDaySeparatorViewDarkPreview_0_null_1,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f35f0d9cfa98f6395349adc75e1ab3d0bcc29f7bf77f04c5c91101a522da3d0
+size 8035

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline.components.virtual_null_DefaultGroup_TimelineItemDaySeparatorViewLightPreview_0_null_0,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23d3d856b318a454217e14cedce5aa4b8ee4bc6bd071b8cbad3cd61de39e839e
+size 5956

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline.components.virtual_null_DefaultGroup_TimelineItemDaySeparatorViewLightPreview_0_null_1,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ede29b6f70d6e0f9f716858d5186e3c093a0cd630abf68ba06fdf8e9460f9cc
+size 7485

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline.components.virtual_null_DefaultGroup_TimelineLoadingMoreIndicatorDarkPreview_0_null,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:303fe8d1d6703c61db623510be1b9b7f0d31c376b80cfe0fbe65b2827ab7a26c
+size 4724

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline.components.virtual_null_DefaultGroup_TimelineLoadingMoreIndicatorLightPreview_0_null,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4be2e216658b7f4613720beff832dd3b489e82182fa30dce005c120e2ff4918a
+size 4517

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_LoginRootScreenDarkPreview_0_null_0,NEXUS_5,1,en].png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7be62fbfef658b54e9a2fea2b66cae87de145f1dd8dd70baa4828d1c2a8c9c96
-size 31637

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_LoginRootScreenDarkPreview_0_null_1,NEXUS_5,1,en].png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ffbfcdf2cd238cd58bac1cbc141be5aa7e74184c4b35018777552d6956e17263
-size 43423

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_LoginRootScreenDarkPreview_0_null_2,NEXUS_5,1,en].png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:49fb267e9136c8b78f083156681b9432c757af755a2161cfdea42df43371e645
-size 33431

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_LoginRootScreenDarkPreview_0_null_3,NEXUS_5,1,en].png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aa2177214c486e5af577c7d001bf92f1fbe7e1b68adcef3c2d7b91fe1d95a864
-size 55533

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_LoginRootScreenDarkPreview_0_null_4,NEXUS_5,1,en].png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ffb69c688191b581c220d2b1b4f89be74a59496caaf39a1e54a3d5ddc586b319
-size 29603

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_LoginRootScreenDarkPreview_0_null_5,NEXUS_5,1,en].png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:461792b0350436f4dd3527243b8a8fb57d619cb94c6f49db93462675c1ada886
-size 51009

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_LoginRootScreenLightPreview_0_null_0,NEXUS_5,1,en].png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3697be1c70d071df9f1af57085d31a44a175b323602bbbe00dea091757d5e06d
-size 31265

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_LoginRootScreenLightPreview_0_null_1,NEXUS_5,1,en].png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:37461ec9e58f5999e5d0bc0810a84131ec5faaec2b7e99359b36f05d8865d3d2
-size 41757

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_LoginRootScreenLightPreview_0_null_2,NEXUS_5,1,en].png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4b4489e2f3f326f952d4680adcb14e88174628454bd3b3503864166074c5ca5c
-size 33177

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_LoginRootScreenLightPreview_0_null_3,NEXUS_5,1,en].png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2ee3a40f5c8131f588642420d4e682f6ed0a00d6faf3d793b249e28c43b09e9c
-size 52914

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_LoginRootScreenLightPreview_0_null_4,NEXUS_5,1,en].png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:154c03f8e6eb6be2cb26fe077b4e6397df13173fe6bfcd3f23bc8287a37921b7
-size 29275

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_LoginRootScreenLightPreview_0_null_5,NEXUS_5,1,en].png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0a9a1a9ac69f15110934ae2de3ce55d7c1d3027079e68ecdf5dcb2861863ce8e
-size 49057

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_TimelineViewDarkPreview_0_null_0,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b22e7a184b107b52d27a193eae0670618e42092536c960b4af230cf39f8e6b9
+size 31588

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_TimelineViewDarkPreview_0_null_1,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adf35653d2a8b133b8f0af348bcc9c88f42ebe8dfe8c2db833815ddd8f11d0f5
+size 43362

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_TimelineViewDarkPreview_0_null_2,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bbe8b0557389a93d84c1b3e9b00ab543837c407483e2ad6af788c9cd4ec5352
+size 33378

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_TimelineViewDarkPreview_0_null_3,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82821019e89520ab76392c79ee090e3cec2453a23ce9b21413c30e6d4473daa4
+size 55473

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_TimelineViewDarkPreview_0_null_4,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06f996b21ed53bf669b3a7a9696f744e604f4e851097ace52900fb47ea46b704
+size 29552

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_TimelineViewDarkPreview_0_null_5,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2199870d0f00ffe8391ceb16566ee97541de34d407c4f8063d3afb53b84824ad
+size 50942

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_TimelineViewLightPreview_0_null_0,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b32ff84ce5b4e79ca70d6eff1074289a89e304640fccb66c1604a5b44001387
+size 31207

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_TimelineViewLightPreview_0_null_1,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d35407c215cf24dd4fcf88b93d4bb1708f28fe7505b2da61c4b1bacca88ae0cf
+size 41701

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_TimelineViewLightPreview_0_null_2,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d97782fa890d055a13b621cdcac6ce62b0842d2adb1ee854f86a09b305c92e58
+size 33122

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_TimelineViewLightPreview_0_null_3,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93e850a532862fb4a5d53bc3af1d3d95f9eb81344d0e74807710a5975d2ee3d9
+size 52853

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_TimelineViewLightPreview_0_null_4,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a09e21db1ae298c058d214ee5c362240f40ccdf8d5ee0ec47d904afe13a95b02
+size 29220

tests/uitests/src/test/snapshots/images/io.element.android.tests.uitests_ScreenshotTest_preview_tests[io.element.android.features.messages.timeline_null_DefaultGroup_TimelineViewLightPreview_0_null_5,NEXUS_5,1,en].png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee2150484bffcca09bcf2151cecdb158cec04220fc53f3f15326b9e8dcd48595
+size 48989