Sulkta-OSS/element-x-ada
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[Media upload] Upload image, video and files (#411)

Browse source 
* Add media upload

* Display media upload error messages using a Snackbar.
This commit is contained in:
Jorge Martin Espinosa 2023-05-11 17:56:13 +02:00 • committed by GitHub
parent 1765398eb1
commit 89b9db3be6
24 changed files with 373 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

5
libraries/androidutils/src/main/kotlin/io/element/android/libraries/androidutils/file/File.kt
View file

@ -37,6 +37,7 @@ fun File.safeDelete() {
)
}
suspend fun Context.createTmpFile(baseDir: File = cacheDir): File = withContext(Dispatchers.IO) {
File.createTempFile(UUID.randomUUID().toString(), null, baseDir).apply { mkdirs() }
suspend fun Context.createTmpFile(baseDir: File = cacheDir, extension: String? = null): File = withContext(Dispatchers.IO) {
val suffix = extension?.let { ".$extension" }
File.createTempFile(UUID.randomUUID().toString(), suffix, baseDir).apply { mkdirs() }
}

1
libraries/core/src/main/kotlin/io/element/android/libraries/core/mimetype/MimeTypes.kt
View file

@ -32,6 +32,7 @@ object MimeTypes {
const val Gif = "image/gif"
const val Videos = "video/*"
const val Mp4 = "video/mp4"
const val Audio = "audio/*"

3
libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/AudioInfo.kt
View file

@ -18,5 +18,6 @@ package io.element.android.libraries.matrix.api.media
data class AudioInfo(
val duration: Long?,
val size: Long?
val size: Long?,
val mimeType: String?,
)

13
libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/room/MatrixRoom.kt
View file

@ -20,10 +20,15 @@ import io.element.android.libraries.matrix.api.core.EventId
import io.element.android.libraries.matrix.api.core.RoomId
import io.element.android.libraries.matrix.api.core.SessionId
import io.element.android.libraries.matrix.api.core.UserId
import io.element.android.libraries.matrix.api.media.AudioInfo
import io.element.android.libraries.matrix.api.media.FileInfo
import io.element.android.libraries.matrix.api.media.ImageInfo
import io.element.android.libraries.matrix.api.media.VideoInfo
import io.element.android.libraries.matrix.api.timeline.MatrixTimeline
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.StateFlow
import java.io.Closeable
import java.io.File
interface MatrixRoom : Closeable {
val sessionId: SessionId
@ -67,6 +72,14 @@ interface MatrixRoom : Closeable {
suspend fun redactEvent(eventId: EventId, reason: String? = null): Result<Unit>
suspend fun sendImage(file: File, thumbnailFile: File, imageInfo: ImageInfo): Result<Unit>
suspend fun sendVideo(file: File, thumbnailFile: File, videoInfo: VideoInfo): Result<Unit>
suspend fun sendAudio(file: File, audioInfo: AudioInfo): Result<Unit>
suspend fun sendFile(file: File, fileInfo: FileInfo): Result<Unit>
suspend fun leave(): Result<Unit>
suspend fun acceptInvitation(): Result<Unit>

9
libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/AudioInfo.kt
View file

@ -21,5 +21,12 @@ import org.matrix.rustcomponents.sdk.AudioInfo as RustAudioInfo
fun RustAudioInfo.map(): AudioInfo = AudioInfo(
duration = duration?.toLong(),
size = size?.toLong()
size = size?.toLong(),
mimeType = mimetype
)
fun AudioInfo.map(): RustAudioInfo = RustAudioInfo(
duration = duration?.toULong(),
size = size?.toULong(),
mimetype = mimeType,
)

7
libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/FileInfo.kt
View file

@ -27,3 +27,10 @@ fun RustFileInfo.map(): FileInfo = FileInfo(
thumbnailInfo = thumbnailInfo?.map(),
thumbnailUrl = thumbnailSource?.useUrl()
)
fun FileInfo.map(): RustFileInfo = RustFileInfo(
mimetype = mimetype,
size = size?.toULong(),
thumbnailInfo = thumbnailInfo?.map(),
thumbnailSource = null
)

11
libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/ImageInfo.kt
View file

@ -17,6 +17,7 @@
package io.element.android.libraries.matrix.impl.media
import io.element.android.libraries.matrix.api.media.ImageInfo
import org.matrix.rustcomponents.sdk.MediaSource
import org.matrix.rustcomponents.sdk.ImageInfo as RustImageInfo
fun RustImageInfo.map(): ImageInfo = ImageInfo(
@ -28,3 +29,13 @@ fun RustImageInfo.map(): ImageInfo = ImageInfo(
thumbnailUrl = thumbnailSource?.useUrl(),
blurhash = blurhash
)
fun ImageInfo.map(): RustImageInfo = RustImageInfo(
height = height?.toULong(),
width = width?.toULong(),
mimetype = mimetype,
size = size?.toULong(),
thumbnailInfo = thumbnailInfo?.map(),
thumbnailSource = null,
blurhash = blurhash,
)

7
libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/ThumbnailInfo.kt
View file

@ -25,3 +25,10 @@ fun RustThumbnailInfo.map(): ThumbnailInfo = ThumbnailInfo(
mimetype = mimetype,
size = size?.toLong()
)
fun ThumbnailInfo.map(): RustThumbnailInfo = RustThumbnailInfo(
height = height?.toULong(),
width = width?.toULong(),
mimetype = mimetype,
size = size?.toULong()
)

11
libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/VideoInfo.kt
View file

@ -29,3 +29,14 @@ fun RustVideoInfo.map(): VideoInfo = VideoInfo(
thumbnailUrl = thumbnailSource?.useUrl(),
blurhash = blurhash
)
fun VideoInfo.map(): RustVideoInfo = RustVideoInfo(
duration = duration?.toULong(),
height = height?.toULong(),
width = width?.toULong(),
mimetype = mimetype,
size = size?.toULong(),
thumbnailInfo = thumbnailInfo?.map(),
thumbnailSource = null,
blurhash = blurhash
)

30
libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/room/RustMatrixRoom.kt
View file

@ -21,10 +21,15 @@ import io.element.android.libraries.matrix.api.core.EventId
import io.element.android.libraries.matrix.api.core.RoomId
import io.element.android.libraries.matrix.api.core.SessionId
import io.element.android.libraries.matrix.api.core.UserId
import io.element.android.libraries.matrix.api.media.AudioInfo
import io.element.android.libraries.matrix.api.media.FileInfo
import io.element.android.libraries.matrix.api.media.ImageInfo
import io.element.android.libraries.matrix.api.media.VideoInfo
import io.element.android.libraries.matrix.api.room.MatrixRoom
import io.element.android.libraries.matrix.api.room.MatrixRoomMembersState
import io.element.android.libraries.matrix.api.room.roomMembers
import io.element.android.libraries.matrix.api.timeline.MatrixTimeline
import io.element.android.libraries.matrix.impl.media.map
import io.element.android.libraries.matrix.impl.timeline.RustMatrixTimeline
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.flow.Flow
@ -39,6 +44,7 @@ import org.matrix.rustcomponents.sdk.SlidingSyncRoom
import org.matrix.rustcomponents.sdk.UpdateSummary
import org.matrix.rustcomponents.sdk.genTransactionId
import org.matrix.rustcomponents.sdk.messageEventContentFromMarkdown
import java.io.File
class RustMatrixRoom(
override val sessionId: SessionId,
@ -202,4 +208,28 @@ class RustMatrixRoom(
}
}
override suspend fun sendImage(file: File, thumbnailFile: File, imageInfo: ImageInfo): Result<Unit> {
return runCatching {
innerRoom.sendImage(file.path, thumbnailFile.path, imageInfo.map())
}
}
override suspend fun sendVideo(file: File, thumbnailFile: File, videoInfo: VideoInfo): Result<Unit> {
return runCatching {
innerRoom.sendVideo(file.path, thumbnailFile.path, videoInfo.map())
}
}
override suspend fun sendAudio(file: File, audioInfo: AudioInfo): Result<Unit> {
return runCatching {
innerRoom.sendAudio(file.path, audioInfo.map())
}
}
override suspend fun sendFile(file: File, fileInfo: FileInfo): Result<Unit> {
return runCatching {
innerRoom.sendFile(file.path, fileInfo.map())
}
}
}

20
libraries/matrix/test/src/main/kotlin/io/element/android/libraries/matrix/test/room/FakeMatrixRoom.kt
View file

@ -20,6 +20,10 @@ import io.element.android.libraries.matrix.api.core.EventId
import io.element.android.libraries.matrix.api.core.RoomId
import io.element.android.libraries.matrix.api.core.SessionId
import io.element.android.libraries.matrix.api.core.UserId
import io.element.android.libraries.matrix.api.media.AudioInfo
import io.element.android.libraries.matrix.api.media.FileInfo
import io.element.android.libraries.matrix.api.media.ImageInfo
import io.element.android.libraries.matrix.api.media.VideoInfo
import io.element.android.libraries.matrix.api.room.MatrixRoom
import io.element.android.libraries.matrix.api.room.MatrixRoomMembersState
import io.element.android.libraries.matrix.api.timeline.MatrixTimeline
@ -30,6 +34,7 @@ import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.emptyFlow
import java.io.File
class FakeMatrixRoom(
override val sessionId: SessionId = A_SESSION_ID,
@ -54,6 +59,9 @@ class FakeMatrixRoom(
private var updateMembersResult: Result<Unit> = Result.success(Unit)
private var acceptInviteResult = Result.success(Unit)
private var rejectInviteResult = Result.success(Unit)
private var sendMediaResult = Result.success(Unit)
var sendMediaCount = 0
private set
var isInviteAccepted: Boolean = false
private set
@ -128,6 +136,14 @@ class FakeMatrixRoom(
return rejectInviteResult
}
override suspend fun sendImage(file: File, thumbnailFile: File, imageInfo: ImageInfo): Result<Unit> = sendMediaResult.also { sendMediaCount++ }
override suspend fun sendVideo(file: File, thumbnailFile: File, videoInfo: VideoInfo): Result<Unit> = sendMediaResult.also { sendMediaCount++ }
override suspend fun sendAudio(file: File, audioInfo: AudioInfo): Result<Unit> = sendMediaResult.also { sendMediaCount++ }
override suspend fun sendFile(file: File, fileInfo: FileInfo): Result<Unit> = sendMediaResult.also { sendMediaCount++ }
override fun close() = Unit
fun givenLeaveRoomError(throwable: Throwable?) {
@ -165,4 +181,8 @@ class FakeMatrixRoom(
fun givenUnIgnoreResult(result: Result<Unit>) {
unignoreResult = result
}
fun givenSendMediaResult(result: Result<Unit>) {
sendMediaResult = result
}
}

5
libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaUploadInfo.kt
View file

@ -24,8 +24,8 @@ import io.element.android.libraries.matrix.api.media.VideoInfo
import java.io.File
sealed interface MediaUploadInfo {
data class Image(val file: File, val info: ImageInfo, val thumbnailInfo: ThumbnailProcessingInfo?) : MediaUploadInfo
data class Video(val file: File, val info: VideoInfo, val thumbnailInfo: ThumbnailProcessingInfo?) : MediaUploadInfo
data class Image(val file: File, val info: ImageInfo, val thumbnailInfo: ThumbnailProcessingInfo) : MediaUploadInfo
data class Video(val file: File, val info: VideoInfo, val thumbnailInfo: ThumbnailProcessingInfo) : MediaUploadInfo
data class Audio(val file: File, val info: AudioInfo) : MediaUploadInfo
data class AnyFile(val file: File, val info: FileInfo) : MediaUploadInfo
}
@ -33,4 +33,5 @@ sealed interface MediaUploadInfo {
data class ThumbnailProcessingInfo(
val file: File,
val info: ThumbnailInfo,
val blurhash: String,
)

1
libraries/mediaupload/impl/build.gradle.kts
View file

@ -41,6 +41,7 @@ android {
implementation(libs.androidx.exifinterface)
implementation(libs.coroutines.core)
implementation(libs.otaliastudios.transcoder)
implementation(libs.vanniktech.blurhash)
testImplementation(libs.test.junit)
testImplementation(libs.coroutines.test)

13
libraries/mediaupload/impl/src/main/kotlin/io/element/android/libraries/mediaupload/ImageCompressor.kt
View file

@ -19,6 +19,7 @@ package io.element.android.libraries.mediaupload
import android.content.Context
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import com.vanniktech.blurhash.BlurHash
import io.element.android.libraries.androidutils.bitmap.calculateInSampleSize
import io.element.android.libraries.androidutils.bitmap.resizeToMax
import io.element.android.libraries.androidutils.bitmap.rotateToMetadataOrientation
@ -48,14 +49,21 @@ class ImageCompressor @Inject constructor(
): Result<ImageCompressionResult> = withContext(Dispatchers.IO) {
runCatching {
val compressedBitmap = compressToBitmap(inputStream, resizeMode).getOrThrow()
val blurhash = BlurHash.encode(compressedBitmap, 3, 3)
// Encode bitmap to the destination temporary file
val tmpFile = context.createTmpFile()
val tmpFile = context.createTmpFile(extension = "jpeg")
tmpFile.outputStream().use {
compressedBitmap.compress(format, desiredQuality, it)
}
ImageCompressionResult(tmpFile, compressedBitmap.width, compressedBitmap.height, tmpFile.length())
ImageCompressionResult(
file = tmpFile,
width = compressedBitmap.width,
height = compressedBitmap.height,
size = tmpFile.length(),
blurhash = blurhash
)
}
}
@ -108,6 +116,7 @@ data class ImageCompressionResult(
val width: Int,
val height: Int,
val size: Long,
val blurhash: String,
)
sealed interface ResizeMode {

57
libraries/mediaupload/impl/src/main/kotlin/io/element/android/libraries/mediaupload/MediaPreProcessorImpl.kt
View file

@ -26,9 +26,7 @@ import io.element.android.libraries.androidutils.file.createTmpFile
import io.element.android.libraries.androidutils.media.runAndRelease
import io.element.android.libraries.core.data.tryOrNull
import io.element.android.libraries.core.mimetype.MimeTypes
import io.element.android.libraries.core.mimetype.MimeTypes.isMimeTypeAudio
import io.element.android.libraries.core.mimetype.MimeTypes.isMimeTypeImage
import io.element.android.libraries.core.mimetype.MimeTypes.isMimeTypeVideo
import io.element.android.libraries.di.AppScope
import io.element.android.libraries.di.ApplicationContext
import io.element.android.libraries.matrix.api.media.AudioInfo
@ -41,7 +39,6 @@ import io.element.android.libraries.mediaupload.api.MediaType
import io.element.android.libraries.mediaupload.api.MediaUploadInfo
import io.element.android.libraries.mediaupload.api.ThumbnailProcessingInfo
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.catch
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.onEach
@ -93,19 +90,23 @@ class MediaPreProcessorImpl @Inject constructor(
deleteOriginal: Boolean,
): Result<MediaUploadInfo> = runCatching {
// Camera returns an 'octet-stream' mimetype, so it needs to be overridden
val originalMimeType = contentResolver.getType(uri)
val mimeType = when (mediaType) {
MediaType.Image -> MimeTypes.Images
MediaType.Video -> MimeTypes.Videos
MediaType.Audio -> MimeTypes.Audio
else -> originalMimeType
val mimeType = contentResolver.getType(uri)
val mimeTypeOrDefault = if (mimeType == MimeTypes.OctetStream) {
when(mediaType) {
MediaType.Image -> MimeTypes.Jpeg
MediaType.Video -> MimeTypes.Mp4
MediaType.Audio -> MimeTypes.Ogg
else -> mimeType
}
} else {
mimeType
}
val compressBeforeSending = mediaType in sequenceOf(MediaType.Image, MediaType.Video)
val result = if (compressBeforeSending && mimeType != MimeTypes.Gif) {
when {
mimeType.isMimeTypeImage() -> processImage(uri)
mimeType.isMimeTypeVideo() -> processVideo(uri, mimeType)
mimeType.isMimeTypeAudio() -> processAudio(uri)
when(mediaType) {
MediaType.Image -> processImage(uri)
MediaType.Video -> processVideo(uri, mimeTypeOrDefault)
MediaType.Audio -> processAudio(uri, mimeTypeOrDefault)
else -> error("Cannot compress file of type: $mimeType")
}
} else {
@ -115,7 +116,7 @@ class MediaPreProcessorImpl @Inject constructor(
removeSensitiveImageMetadata(file)
}
val info = FileInfo(
mimetype = originalMimeType,
mimetype = mimeType,
size = file.length(),
thumbnailInfo = null,
thumbnailUrl = null,
@ -141,7 +142,7 @@ class MediaPreProcessorImpl @Inject constructor(
removeSensitiveImageMetadata(compressedFileResult.file)
val thumbnailResult = compressedFileResult.file.inputStream().use { generateImageThumbnail(it) }
val processingResult = compressedFileResult.toImageInfo(MimeTypes.Jpeg, thumbnailResult?.file?.path, thumbnailResult?.info)
val processingResult = compressedFileResult.toImageInfo(MimeTypes.Jpeg, thumbnailResult.file.path, thumbnailResult.info)
return MediaUploadInfo.Image(compressedFileResult.file, processingResult, thumbnailResult)
}
@ -155,32 +156,33 @@ class MediaPreProcessorImpl @Inject constructor(
.first()
.file
val videoProcessingInfo = extractVideoMetadata(resultFile, mimeType, thumbnailInfo?.file?.path, thumbnailInfo?.info)
val videoProcessingInfo = extractVideoMetadata(resultFile, mimeType, thumbnailInfo.file.path, thumbnailInfo)
return MediaUploadInfo.Video(resultFile, videoProcessingInfo, thumbnailInfo)
}
private suspend fun processAudio(uri: Uri): MediaUploadInfo {
private suspend fun processAudio(uri: Uri, mimeType: String?): MediaUploadInfo {
val file = copyToTmpFile(uri)
return MediaMetadataRetriever().runAndRelease {
setDataSource(context, Uri.fromFile(file))
val info = AudioInfo(
duration = extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)?.toLong() ?: 0L,
size = file.length()
size = file.length(),
mimeType = mimeType,
)
MediaUploadInfo.Audio(file, info)
}
}
private suspend fun generateImageThumbnail(inputStream: InputStream): ThumbnailProcessingInfo? {
private suspend fun generateImageThumbnail(inputStream: InputStream): ThumbnailProcessingInfo {
val thumbnailResult = imageCompressor
.compressToTmpFile(
inputStream = inputStream,
resizeMode = ResizeMode.Strict(THUMB_MAX_WIDTH, THUMB_MAX_HEIGHT),
).getOrNull()
).getOrThrow()
return thumbnailResult?.toThumbnailProcessingInfo(MimeTypes.Jpeg)
return thumbnailResult.toThumbnailProcessingInfo(MimeTypes.Jpeg)
}
private fun removeSensitiveImageMetadata(file: File) {
@ -203,7 +205,7 @@ class MediaPreProcessorImpl @Inject constructor(
}
}
private fun extractVideoMetadata(file: File, mimeType: String?, thumbnailUrl: String?, thumbnailInfo: ThumbnailInfo?): VideoInfo =
private fun extractVideoMetadata(file: File, mimeType: String?, thumbnailUrl: String?, thumbnailInfo: ThumbnailProcessingInfo?): VideoInfo =
MediaMetadataRetriever().runAndRelease {
setDataSource(context, Uri.fromFile(file))
@ -213,16 +215,16 @@ class MediaPreProcessorImpl @Inject constructor(
height = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toLong() ?: 0L,
mimetype = mimeType,
size = file.length(),
thumbnailInfo = thumbnailInfo,
thumbnailInfo = thumbnailInfo?.info,
thumbnailUrl = thumbnailUrl,
blurhash = null,
blurhash = thumbnailInfo?.blurhash,
)
}
private suspend fun extractVideoThumbnail(uri: Uri): ThumbnailProcessingInfo? =
private suspend fun extractVideoThumbnail(uri: Uri): ThumbnailProcessingInfo =
MediaMetadataRetriever().runAndRelease {
setDataSource(context, uri)
val bitmap = getFrameAtTime(VIDEO_THUMB_FRAME) ?: return@runAndRelease null
val bitmap = requireNotNull(getFrameAtTime(VIDEO_THUMB_FRAME))
val inputStream = ByteArrayOutputStream().use {
bitmap.compress(Bitmap.CompressFormat.JPEG, 80, it)
ByteArrayInputStream(it.toByteArray())
@ -249,7 +251,7 @@ fun ImageCompressionResult.toImageInfo(mimeType: String, thumbnailUrl: String?,
size = size,
thumbnailInfo = thumbnailInfo,
thumbnailUrl = thumbnailUrl,
blurhash = null,
blurhash = blurhash,
)
fun ImageCompressionResult.toThumbnailProcessingInfo(mimeType: String) = ThumbnailProcessingInfo(
@ -260,4 +262,5 @@ fun ImageCompressionResult.toThumbnailProcessingInfo(mimeType: String) = Thumbna
mimetype = mimeType,
size = size,
),
blurhash = blurhash,
)

2
libraries/mediaupload/impl/src/main/kotlin/io/element/android/libraries/mediaupload/VideoCompressor.kt
View file

@ -32,7 +32,7 @@ class VideoCompressor @Inject constructor(
) {
fun compress(uri: Uri) = callbackFlow {
val tmpFile = context.createTmpFile()
val tmpFile = context.createTmpFile(extension = "mp4")
val future = Transcoder.into(tmpFile.path)
.addDataSource(context, uri)
.setListener(object : TranscoderListener {