Merge pull request #1992 from vector-im/feature/bma/notificationUserAvatar

Render images in notification

changelog.d/1991.misc

@@ -0,0 +1 @@
+Render images in Notification

features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/timeline/VoiceMessageMediaRepo.kt

@@ -24,6 +24,7 @@ import io.element.android.libraries.di.CacheDirectory
 import io.element.android.libraries.di.RoomScope
 import io.element.android.libraries.matrix.api.media.MatrixMediaLoader
 import io.element.android.libraries.matrix.api.media.MediaSource
+import io.element.android.libraries.matrix.api.mxc.MxcTools
 import java.io.File
 
 /**
@@ -66,6 +67,7 @@ interface VoiceMessageMediaRepo {
 
 class DefaultVoiceMessageMediaRepo @AssistedInject constructor(
     @CacheDirectory private val cacheDir: File,
+    mxcTools: MxcTools,
     private val matrixMediaLoader: MatrixMediaLoader,
     @Assisted private val mediaSource: MediaSource,
     @Assisted("mimeType") private val mimeType: String?,
@@ -101,7 +103,7 @@ class DefaultVoiceMessageMediaRepo @AssistedInject constructor(
         }
     }
 
-    private val cachedFile: File? = mxcUri2FilePath(mediaSource.url)?.let {
+    private val cachedFile: File? = mxcTools.mxcUri2FilePath(mediaSource.url)?.let {
         File("${cacheDir.path}/$CACHE_VOICE_SUBDIR/$it")
     }
 }
@@ -110,24 +112,3 @@ class DefaultVoiceMessageMediaRepo @AssistedInject constructor(
  * Subdirectory of the application's cache directory where voice messages are stored.
  */
 private const val CACHE_VOICE_SUBDIR = "temp/voice"
-
-/**
- * Regex to match a Matrix Content (mxc://) URI.
- *
- * See: https://spec.matrix.org/v1.8/client-server-api/#matrix-content-mxc-uris
- */
-private val mxcRegex = Regex("""^mxc:\/\/([^\/]+)\/([^\/]+)$""")
-
-/**
- * Sanitizes an mxcUri to be used as a relative file path.
- *
- * @param mxcUri the Matrix Content (mxc://) URI of the voice message.
- * @return the relative file path as "<server-name>/<media-id>" or null if the mxcUri is invalid.
- */
-private fun mxcUri2FilePath(mxcUri: String): String? = mxcRegex.matchEntire(mxcUri)?.let { match ->
-    buildString {
-        append(match.groupValues[1])
-        append("/")
-        append(match.groupValues[2])
-    }
-}

features/messages/impl/src/test/kotlin/io/element/android/features/messages/impl/voicemessages/timeline/DefaultVoiceMessageMediaRepoTest.kt

@@ -20,6 +20,7 @@ import com.google.common.truth.Truth.assertThat
 import io.element.android.libraries.core.mimetype.MimeTypes
 import io.element.android.libraries.matrix.api.media.MatrixMediaLoader
 import io.element.android.libraries.matrix.api.media.MediaSource
+import io.element.android.libraries.matrix.api.mxc.MxcTools
 import io.element.android.libraries.matrix.test.media.FakeMediaLoader
 import kotlinx.coroutines.test.runTest
 import org.junit.Rule
@@ -139,6 +140,7 @@ private fun createDefaultVoiceMessageMediaRepo(
     mxcUri: String = MXC_URI,
 ) = DefaultVoiceMessageMediaRepo(
     cacheDir = temporaryFolder.root,
+    mxcTools = MxcTools(),
     matrixMediaLoader = matrixMediaLoader,
     mediaSource = MediaSource(
         url = mxcUri,

libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/mxc/MxcTools.kt

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2023 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.element.android.libraries.matrix.api.mxc
+
+import javax.inject.Inject
+
+class MxcTools @Inject constructor() {
+    /**
+     * Regex to match a Matrix Content (mxc://) URI.
+     *
+     * See: https://spec.matrix.org/v1.8/client-server-api/#matrix-content-mxc-uris
+     */
+    private val mxcRegex = Regex("""^mxc://([^/]+)/([^/]+)$""")
+
+    /**
+     * Sanitizes an mxcUri to be used as a relative file path.
+     *
+     * @param mxcUri the Matrix Content (mxc://) URI of the file.
+     * @return the relative file path as "<server-name>/<media-id>" or null if the mxcUri is invalid.
+     */
+    fun mxcUri2FilePath(mxcUri: String): String? = mxcRegex.matchEntire(mxcUri)?.let { match ->
+        buildString {
+            append(match.groupValues[1])
+            append("/")
+            append(match.groupValues[2])
+        }
+    }
+}

libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/notification/NotificationData.kt

@@ -25,6 +25,7 @@ import io.element.android.libraries.matrix.api.timeline.item.event.MessageType
 data class NotificationData(
     val eventId: EventId,
     val roomId: RoomId,
+    // mxc url
     val senderAvatarUrl: String?,
     val senderDisplayName: String?,
     val roomAvatarUrl: String?,
@@ -34,8 +35,6 @@ data class NotificationData(
     val isNoisy: Boolean,
     val timestamp: Long,
     val content: NotificationContent,
-    // For images for instance
-    val contentUrl: String?,
     val hasMention: Boolean,
 )
 

libraries/matrix/api/src/test/kotlin/io/element/android/libraries/matrix/api/mxc/MxcToolsTest.kt

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2023 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.element.android.libraries.matrix.api.mxc
+
+import com.google.common.truth.Truth.assertThat
+import org.junit.Test
+
+class MxcToolsTest {
+    @Test
+    fun `mxcUri2FilePath returns extracted path`() {
+        val mxcTools = MxcTools()
+        val mxcUri = "mxc://server.org/abc123"
+        val filePath = mxcTools.mxcUri2FilePath(mxcUri)
+        assertThat(filePath).isEqualTo("server.org/abc123")
+    }
+
+    @Test
+    fun `mxcUri2FilePath returns null for invalid data`() {
+        val mxcTools = MxcTools()
+        assertThat(mxcTools.mxcUri2FilePath("")).isNull()
+        assertThat(mxcTools.mxcUri2FilePath("mxc://server.org")).isNull()
+        assertThat(mxcTools.mxcUri2FilePath("mxc://server.org/")).isNull()
+        assertThat(mxcTools.mxcUri2FilePath("m://server.org/abc123")).isNull()
+    }
+}

libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/notification/NotificationMapper.kt

@@ -52,7 +52,6 @@ class NotificationMapper(
                 isNoisy = item.isNoisy.orFalse(),
                 timestamp = item.timestamp() ?: clock.epochMillis(),
                 content = item.event.use { notificationContentMapper.map(it) },
-                contentUrl = null,
                 hasMention = item.hasMention.orFalse(),
             )
         }

libraries/push/impl/src/main/AndroidManifest.xml

@@ -15,6 +15,7 @@
 -->
 
 <manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
     <uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
     <application>
         <receiver
@@ -24,5 +25,17 @@
             android:name=".notifications.NotificationBroadcastReceiver"
             android:enabled="true"
             android:exported="false" />
+
+        <provider
+            android:name=".notifications.NotificationsFileProvider"
+            android:authorities="${applicationId}.notifications.fileprovider"
+            android:exported="false"
+            android:grantUriPermissions="true">
+            <meta-data
+                android:name="android.support.FILE_PROVIDER_PATHS"
+                android:resource="@xml/notifications_provider_paths" />
+        </provider>
+
     </application>
+
 </manifest>

libraries/push/impl/src/main/kotlin/io/element/android/libraries/push/impl/notifications/NotifiableEventResolver.kt

@@ -16,7 +16,12 @@
 
 package io.element.android.libraries.push.impl.notifications
 
+import android.content.Context
+import android.net.Uri
+import androidx.core.content.FileProvider
 import io.element.android.libraries.core.log.logger.LoggerTag
+import io.element.android.libraries.di.ApplicationContext
+import io.element.android.libraries.matrix.api.MatrixClient
 import io.element.android.libraries.matrix.api.MatrixClientProvider
 import io.element.android.libraries.matrix.api.core.EventId
 import io.element.android.libraries.matrix.api.core.RoomId
@@ -60,6 +65,8 @@ class NotifiableEventResolver @Inject constructor(
     private val stringProvider: StringProvider,
     private val clock: SystemClock,
     private val matrixClientProvider: MatrixClientProvider,
+    private val notificationMediaRepoFactory: NotificationMediaRepo.Factory,
+    @ApplicationContext private val context: Context,
 ) {
 
     suspend fun resolveEvent(sessionId: SessionId, roomId: RoomId, eventId: EventId): NotifiableEvent? {
@@ -75,10 +82,13 @@ class NotifiableEventResolver @Inject constructor(
         }.getOrNull()
 
         // TODO this notificationData is not always valid at the moment, sometimes the Rust SDK can't fetch the matching event
-        return notificationData?.asNotifiableEvent(sessionId)
+        return notificationData?.asNotifiableEvent(client, sessionId)
     }
 
-    private fun NotificationData.asNotifiableEvent(userId: SessionId): NotifiableEvent? {
+    private suspend fun NotificationData.asNotifiableEvent(
+        client: MatrixClient,
+        userId: SessionId,
+    ): NotifiableEvent? {
         return when (val content = this.content) {
             is NotificationContent.MessageLike.RoomMessage -> {
                 val messageBody = descriptionFromMessageContent(content, senderDisplayName ?: content.senderId.value)
@@ -96,7 +106,7 @@ class NotifiableEventResolver @Inject constructor(
                     timestamp = this.timestamp,
                     senderName = senderDisplayName,
                     body = notificationBody,
-                    imageUriString = this.contentUrl,
+                    imageUriString = fetchImageIfPresent(client)?.toString(),
                     roomName = roomDisplayName,
                     roomIsDirect = isDirect,
                     roomAvatarPath = roomAvatarUrl,
@@ -238,6 +248,34 @@ class NotifiableEventResolver @Inject constructor(
             stringProvider.getString(R.string.notification_room_invite_body)
         }
     }
+
+    private suspend fun NotificationData.fetchImageIfPresent(client: MatrixClient): Uri? {
+        val fileResult = when (val content = this.content) {
+            is NotificationContent.MessageLike.RoomMessage -> {
+                when (val messageType = content.messageType) {
+                    is ImageMessageType -> notificationMediaRepoFactory.create(client)
+                        .getMediaFile(
+                            mediaSource = messageType.source,
+                            mimeType = messageType.info?.mimetype,
+                            body = messageType.body,
+                        )
+                    is VideoMessageType -> null // Use the thumbnail here?
+                    else -> null
+                }
+            }
+            else -> null
+        } ?: return null
+
+        return fileResult
+            .onFailure {
+                Timber.tag(loggerTag.value).e(it, "Failed to download image for notification")
+            }
+            .map { mediaFile ->
+                val authority = "${context.packageName}.notifications.fileprovider"
+                FileProvider.getUriForFile(context, authority, mediaFile)
+            }
+            .getOrNull()
+    }
 }
 
 @Suppress("LongParameterList")

libraries/push/impl/src/main/kotlin/io/element/android/libraries/push/impl/notifications/NotificationMediaRepo.kt

@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2023 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.element.android.libraries.push.impl.notifications
+
+import com.squareup.anvil.annotations.ContributesBinding
+import dagger.assisted.Assisted
+import dagger.assisted.AssistedFactory
+import dagger.assisted.AssistedInject
+import io.element.android.libraries.di.AppScope
+import io.element.android.libraries.di.CacheDirectory
+import io.element.android.libraries.matrix.api.MatrixClient
+import io.element.android.libraries.matrix.api.media.MediaSource
+import io.element.android.libraries.matrix.api.mxc.MxcTools
+import java.io.File
+
+/**
+ * Fetches the media file for a notification.
+ *
+ * Media is downloaded from the rust sdk and stored in the application's cache directory.
+ * Media files are indexed by their Matrix Content (mxc://) URI and considered immutable.
+ * Whenever a given mxc is found in the cache, it is returned immediately.
+ */
+interface NotificationMediaRepo {
+
+    /**
+     * Factory for [NotificationMediaRepo].
+     */
+    fun interface Factory {
+        /**
+         * Creates a [NotificationMediaRepo].
+         *
+         */
+        fun create(
+            client: MatrixClient
+        ): NotificationMediaRepo
+    }
+
+    /**
+     * Returns the file.
+     *
+     * In case of a cache hit the file is returned immediately.
+     * In case of a cache miss the file is downloaded and then returned.
+     *
+     * @param mediaSource the media source of the media.
+     * @param mimeType the mime type of the media.
+     * @param body the body of the message.
+     * @return A [Result] holding either the media [File] from the cache directory or an [Exception].
+     */
+    suspend fun getMediaFile(
+        mediaSource: MediaSource,
+        mimeType: String?,
+        body: String?,
+    ): Result<File>
+}
+
+class DefaultNotificationMediaRepo @AssistedInject constructor(
+    @CacheDirectory private val cacheDir: File,
+    private val mxcTools: MxcTools,
+    @Assisted private val client: MatrixClient,
+) : NotificationMediaRepo {
+
+    @ContributesBinding(AppScope::class)
+    @AssistedFactory
+    fun interface Factory : NotificationMediaRepo.Factory {
+        override fun create(
+            client: MatrixClient,
+        ): DefaultNotificationMediaRepo
+    }
+
+    private val matrixMediaLoader = client.mediaLoader
+
+    override suspend fun getMediaFile(
+        mediaSource: MediaSource,
+        mimeType: String?,
+        body: String?,
+    ): Result<File> {
+        val cachedFile = mediaSource.cachedFile()
+        return when {
+            cachedFile == null -> Result.failure(IllegalStateException("Invalid mxcUri."))
+            cachedFile.exists() -> Result.success(cachedFile)
+            else -> matrixMediaLoader.downloadMediaFile(
+                source = mediaSource,
+                mimeType = mimeType,
+                body = body,
+            ).mapCatching {
+                it.use { mediaFile ->
+                    val dest = cachedFile.apply { parentFile?.mkdirs() }
+                    if (mediaFile.persist(dest.path)) {
+                        dest
+                    } else {
+                        error("Failed to move file to cache.")
+                    }
+                }
+            }
+        }
+    }
+
+    private fun MediaSource.cachedFile(): File? = mxcTools.mxcUri2FilePath(url)?.let {
+        File("${cacheDir.path}/$CACHE_NOTIFICATION_SUBDIR/$it")
+    }
+}
+
+/**
+ * Subdirectory of the application's cache directory where file are stored.
+ */
+private const val CACHE_NOTIFICATION_SUBDIR = "temp/notif"

libraries/push/impl/src/main/kotlin/io/element/android/libraries/push/impl/notifications/NotificationsFileProvider.kt

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2023 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.element.android.libraries.push.impl.notifications
+
+import androidx.core.content.FileProvider
+
+/**
+ * We have to declare our own file provider to avoid collision with other modules
+ * having their own.
+ */
+class NotificationsFileProvider : FileProvider()

libraries/push/impl/src/main/kotlin/io/element/android/libraries/push/impl/notifications/TestNotificationReceiver.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020 New Vector Ltd
+ * Copyright (c) 2023 New Vector Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

libraries/push/impl/src/main/kotlin/io/element/android/libraries/push/impl/notifications/model/NotifiableMessageEvent.kt

@@ -58,9 +58,8 @@ data class NotifiableMessageEvent(
     override val description: String = body ?: ""
     val title: String = senderName ?: ""
 
-    // TODO EAx The image has to be downloaded and expose using the file provider.
+    // Example of value:
-    // Example of value from Element Android:
+    // content://io.element.android.x.debug.notifications.fileprovider/downloads/temp/notif/matrix.org/XGItzSDOnSyXjYtOPfiKexDJ
-    // content://im.vector.app.debug.mx-sdk.fileprovider/downloads/downloads/816abf76d806c768760568952b1862c8/F/72c33edd23dee3b95f4d5a18aa25fa54/image.png
     val imageUri: Uri?
         get() = imageUriString?.let { Uri.parse(it) }
 }

libraries/push/impl/src/main/res/xml/notifications_provider_paths.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<paths>
+    <cache-path
+        name="downloads"
+        path="/" />
+</paths>

libraries/push/impl/src/test/kotlin/io/element/android/libraries/push/impl/notifications/NotifiableEventResolverTest.kt

@@ -42,6 +42,7 @@ import io.element.android.libraries.matrix.test.A_USER_ID_2
 import io.element.android.libraries.matrix.test.FakeMatrixClient
 import io.element.android.libraries.matrix.test.FakeMatrixClientProvider
 import io.element.android.libraries.matrix.test.notification.FakeNotificationService
+import io.element.android.libraries.push.impl.notifications.fake.FakeNotificationMediaRepo
 import io.element.android.libraries.push.impl.notifications.model.FallbackNotifiableEvent
 import io.element.android.libraries.push.impl.notifications.model.InviteNotifiableEvent
 import io.element.android.libraries.push.impl.notifications.model.NotifiableMessageEvent
@@ -257,7 +258,7 @@ class NotifiableEventResolverTest {
                 createNotificationData(
                     content = NotificationContent.MessageLike.RoomMessage(
                         senderId = A_USER_ID_2,
-                        messageType =  LocationMessageType("Location", "geo:1,2", null),
+                        messageType = LocationMessageType("Location", "geo:1,2", null),
                     )
                 )
             )
@@ -274,7 +275,7 @@ class NotifiableEventResolverTest {
                 createNotificationData(
                     content = NotificationContent.MessageLike.RoomMessage(
                         senderId = A_USER_ID_2,
-                        messageType =  NoticeMessageType("Notice", null),
+                        messageType = NoticeMessageType("Notice", null),
                     )
                 )
             )
@@ -291,7 +292,7 @@ class NotifiableEventResolverTest {
                 createNotificationData(
                     content = NotificationContent.MessageLike.RoomMessage(
                         senderId = A_USER_ID_2,
-                        messageType =  EmoteMessageType("is happy", null),
+                        messageType = EmoteMessageType("is happy", null),
                     )
                 )
             )
@@ -487,11 +488,15 @@ class NotifiableEventResolverTest {
                 Result.success(FakeMatrixClient(notificationService = notificationService))
             }
         })
-
+        val notificationMediaRepoFactory = NotificationMediaRepo.Factory {
+            FakeNotificationMediaRepo()
+        }
         return NotifiableEventResolver(
             stringProvider = AndroidStringProvider(context.resources),
             clock = FakeSystemClock(),
             matrixClientProvider = matrixClientProvider,
+            notificationMediaRepoFactory = notificationMediaRepoFactory,
+            context = context,
         )
     }
 
@@ -512,7 +517,6 @@ class NotifiableEventResolverTest {
             isNoisy = false,
             timestamp = A_TIMESTAMP,
             content = content,
-            contentUrl = null,
             hasMention = hasMention,
         )
     }

libraries/push/impl/src/test/kotlin/io/element/android/libraries/push/impl/notifications/fake/FakeNotificationMediaRepo.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2023 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.element.android.libraries.push.impl.notifications.fake
+
+import io.element.android.libraries.matrix.api.media.MediaSource
+import io.element.android.libraries.push.impl.notifications.NotificationMediaRepo
+import java.io.File
+
+class FakeNotificationMediaRepo : NotificationMediaRepo {
+    override suspend fun getMediaFile(
+        mediaSource: MediaSource,
+        mimeType: String?,
+        body: String?,
+    ): Result<File> {
+        return Result.failure(IllegalStateException("Fake class"))
+    }
+}