Fix MediaPreProcessor for images/videos sent as file

This commit is contained in:
ganfra 2023-06-16 13:52:07 +02:00
parent dcb9bcd11e
commit b6e7228ce0
7 changed files with 251 additions and 147 deletions

View file

@ -50,7 +50,7 @@ import io.element.android.libraries.mediapickers.test.FakePickerProvider
import io.element.android.libraries.mediaupload.api.MediaPreProcessor import io.element.android.libraries.mediaupload.api.MediaPreProcessor
import io.element.android.libraries.mediaupload.api.MediaSender import io.element.android.libraries.mediaupload.api.MediaSender
import io.element.android.libraries.mediaupload.api.MediaUploadInfo import io.element.android.libraries.mediaupload.api.MediaUploadInfo
import io.element.android.libraries.mediaupload.api.ThumbnailProcessingInfo import io.element.android.libraries.mediaupload.api.ThumbnailResult
import io.element.android.libraries.mediaupload.test.FakeMediaPreProcessor import io.element.android.libraries.mediaupload.test.FakeMediaPreProcessor
import io.element.android.libraries.textcomposer.MessageComposerMode import io.element.android.libraries.textcomposer.MessageComposerMode
import io.mockk.mockk import io.mockk.mockk
@ -301,7 +301,7 @@ class MessageComposerPresenterTest {
thumbnailSource = null, thumbnailSource = null,
blurhash = null, blurhash = null,
), ),
thumbnailInfo = ThumbnailProcessingInfo( thumbnailInfo = ThumbnailResult(
file = File("/some/path"), file = File("/some/path"),
info = ThumbnailInfo( info = ThumbnailInfo(
width = null, width = null,
@ -309,7 +309,6 @@ class MessageComposerPresenterTest {
mimetype = null, mimetype = null,
size = null, size = null,
), ),
blurhash = "",
) )
) )
) )
@ -344,7 +343,7 @@ class MessageComposerPresenterTest {
thumbnailSource = null, thumbnailSource = null,
blurhash = null, blurhash = null,
), ),
thumbnailInfo = ThumbnailProcessingInfo( thumbnailInfo = ThumbnailResult(
file = File("/some/path"), file = File("/some/path"),
info = ThumbnailInfo( info = ThumbnailInfo(
width = null, width = null,
@ -352,7 +351,6 @@ class MessageComposerPresenterTest {
mimetype = null, mimetype = null,
size = null, size = null,
), ),
blurhash = "",
) )
) )
) )

View file

@ -20,5 +20,9 @@ import android.media.MediaMetadataRetriever
/** [MediaMetadataRetriever] only implements `AutoClosable` since API 29, so we need to execute this to have the same in older APIs. */ /** [MediaMetadataRetriever] only implements `AutoClosable` since API 29, so we need to execute this to have the same in older APIs. */
inline fun <T> MediaMetadataRetriever.runAndRelease(block: MediaMetadataRetriever.() -> T): T { inline fun <T> MediaMetadataRetriever.runAndRelease(block: MediaMetadataRetriever.() -> T): T {
return block().also { release() } return try {
block()
} finally {
release()
}
} }

View file

@ -44,15 +44,26 @@ class MediaSender @Inject constructor(
): Result<Unit> { ): Result<Unit> {
return when (info) { return when (info) {
is MediaUploadInfo.Image -> { is MediaUploadInfo.Image -> {
sendImage(info.file, info.thumbnailInfo.file, info.info) sendImage(
file = info.file,
thumbnailFile = info.thumbnailFile,
imageInfo = info.info
)
} }
is MediaUploadInfo.Video -> { is MediaUploadInfo.Video -> {
sendVideo(info.file, info.thumbnailInfo.file, info.info) sendVideo(
file = info.file,
thumbnailFile = info.thumbnailFile,
videoInfo = info.info
)
} }
is MediaUploadInfo.AnyFile -> { is MediaUploadInfo.AnyFile -> {
sendFile(info.file, info.info) sendFile(
file = info.file,
fileInfo = info.info
)
} }
else -> Result.failure(IllegalStateException("Unexpected MediaUploadInfo format: $info")) else -> Result.failure(IllegalStateException("Unexpected MediaUploadInfo format: $info"))
} }

View file

@ -19,7 +19,6 @@ package io.element.android.libraries.mediaupload.api
import io.element.android.libraries.matrix.api.media.AudioInfo import io.element.android.libraries.matrix.api.media.AudioInfo
import io.element.android.libraries.matrix.api.media.FileInfo import io.element.android.libraries.matrix.api.media.FileInfo
import io.element.android.libraries.matrix.api.media.ImageInfo import io.element.android.libraries.matrix.api.media.ImageInfo
import io.element.android.libraries.matrix.api.media.ThumbnailInfo
import io.element.android.libraries.matrix.api.media.VideoInfo import io.element.android.libraries.matrix.api.media.VideoInfo
import java.io.File import java.io.File
@ -27,14 +26,8 @@ sealed interface MediaUploadInfo {
val file: File val file: File
data class Image(override val file: File, val info: ImageInfo, val thumbnailInfo: ThumbnailProcessingInfo) : MediaUploadInfo data class Image(override val file: File, val info: ImageInfo, val thumbnailFile: File) : MediaUploadInfo
data class Video(override val file: File, val info: VideoInfo, val thumbnailInfo: ThumbnailProcessingInfo) : MediaUploadInfo data class Video(override val file: File, val info: VideoInfo, val thumbnailFile: File) : MediaUploadInfo
data class Audio(override val file: File, val info: AudioInfo) : MediaUploadInfo data class Audio(override val file: File, val info: AudioInfo) : MediaUploadInfo
data class AnyFile(override val file: File, val info: FileInfo) : MediaUploadInfo data class AnyFile(override val file: File, val info: FileInfo) : MediaUploadInfo
} }
data class ThumbnailProcessingInfo(
val file: File,
val info: ThumbnailInfo,
val blurhash: String,
)

View file

@ -17,7 +17,7 @@
package io.element.android.libraries.mediaupload package io.element.android.libraries.mediaupload
import android.content.Context import android.content.Context
import android.graphics.Bitmap import android.graphics.BitmapFactory
import android.media.MediaMetadataRetriever import android.media.MediaMetadataRetriever
import android.net.Uri import android.net.Uri
import androidx.exifinterface.media.ExifInterface import androidx.exifinterface.media.ExifInterface
@ -37,26 +37,21 @@ import io.element.android.libraries.di.ApplicationContext
import io.element.android.libraries.matrix.api.media.AudioInfo import io.element.android.libraries.matrix.api.media.AudioInfo
import io.element.android.libraries.matrix.api.media.FileInfo import io.element.android.libraries.matrix.api.media.FileInfo
import io.element.android.libraries.matrix.api.media.ImageInfo import io.element.android.libraries.matrix.api.media.ImageInfo
import io.element.android.libraries.matrix.api.media.MediaSource
import io.element.android.libraries.matrix.api.media.ThumbnailInfo
import io.element.android.libraries.matrix.api.media.VideoInfo import io.element.android.libraries.matrix.api.media.VideoInfo
import io.element.android.libraries.mediaupload.api.MediaPreProcessor import io.element.android.libraries.mediaupload.api.MediaPreProcessor
import io.element.android.libraries.mediaupload.api.MediaUploadInfo import io.element.android.libraries.mediaupload.api.MediaUploadInfo
import io.element.android.libraries.mediaupload.api.ThumbnailProcessingInfo
import kotlinx.coroutines.flow.filterIsInstance import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.io.File import java.io.File
import java.io.InputStream import java.io.InputStream
import javax.inject.Inject import javax.inject.Inject
import kotlin.time.Duration.Companion.seconds
@ContributesBinding(AppScope::class) @ContributesBinding(AppScope::class)
class AndroidMediaPreProcessor @Inject constructor( class AndroidMediaPreProcessor @Inject constructor(
@ApplicationContext private val context: Context, @ApplicationContext private val context: Context,
private val thumbnailFactory: ThumbnailFactory,
private val imageCompressor: ImageCompressor, private val imageCompressor: ImageCompressor,
private val videoCompressor: VideoCompressor, private val videoCompressor: VideoCompressor,
private val coroutineDispatchers: CoroutineDispatchers, private val coroutineDispatchers: CoroutineDispatchers,
@ -69,23 +64,6 @@ class AndroidMediaPreProcessor @Inject constructor(
* values may surpass this limit. (i.e.: an image of `480x3000px` would have `inSampleSize=1` and be sent as is). * values may surpass this limit. (i.e.: an image of `480x3000px` would have `inSampleSize=1` and be sent as is).
*/ */
private const val IMAGE_SCALE_REF_SIZE = 640 private const val IMAGE_SCALE_REF_SIZE = 640
/**
* Max width of thumbnail images.
* See [the Matrix spec](https://spec.matrix.org/latest/client-server-api/?ref=blog.gitter.im#thumbnails).
*/
private const val THUMB_MAX_WIDTH = 800
/**
* Max height of thumbnail images.
* See [the Matrix spec](https://spec.matrix.org/latest/client-server-api/?ref=blog.gitter.im#thumbnails).
*/
private const val THUMB_MAX_HEIGHT = 600
/**
* Frame of the video to be used for generating a thumbnail.
*/
private val VIDEO_THUMB_FRAME = 5.seconds.inWholeMicroseconds
} }
private val contentResolver = context.contentResolver private val contentResolver = context.contentResolver
@ -95,40 +73,34 @@ class AndroidMediaPreProcessor @Inject constructor(
mimeType: String, mimeType: String,
deleteOriginal: Boolean, deleteOriginal: Boolean,
compressIfPossible: Boolean, compressIfPossible: Boolean,
): Result<MediaUploadInfo> = runCatching { ): Result<MediaUploadInfo> = withContext(coroutineDispatchers.computation) {
val shouldBeCompressed = compressIfPossible && runCatching {
(mimeType.isMimeTypeImage() && mimeType != MimeTypes.Gif) || val result = when {
mimeType.isMimeTypeVideo() mimeType.isMimeTypeImage() -> processImage(uri, mimeType, compressIfPossible && mimeType != MimeTypes.Gif)
mimeType.isMimeTypeVideo() -> processVideo(uri, mimeType, compressIfPossible)
val result = if (shouldBeCompressed) {
when {
mimeType.isMimeTypeImage() -> processImage(uri)
mimeType.isMimeTypeVideo() -> processVideo(uri, mimeType)
mimeType.isMimeTypeAudio() -> processAudio(uri, mimeType) mimeType.isMimeTypeAudio() -> processAudio(uri, mimeType)
else -> error("Cannot compress file of type: $mimeType") else -> processFile(uri, mimeType)
} }
} else { if (deleteOriginal) {
val file = copyToTmpFile(uri) tryOrNull {
// Remove image metadata here too contentResolver.delete(uri, null, null)
if (mimeType.isMimeTypeImage() && mimeType != MimeTypes.Gif) { }
removeSensitiveImageMetadata(file)
} }
val info = FileInfo( result.postProcess(uri)
mimetype = mimeType,
size = file.length(),
thumbnailInfo = null,
thumbnailSource = null,
)
MediaUploadInfo.AnyFile(file, info)
} }
if (deleteOriginal) {
tryOrNull {
contentResolver.delete(uri, null, null)
}
}
result.postProcess(uri)
}.mapFailure { MediaPreProcessor.Failure(it) } }.mapFailure { MediaPreProcessor.Failure(it) }
private suspend fun processFile(uri: Uri, mimeType: String): MediaUploadInfo {
val file = copyToTmpFile(uri)
val info = FileInfo(
mimetype = mimeType,
size = file.length(),
thumbnailInfo = null,
thumbnailSource = null,
)
return MediaUploadInfo.AnyFile(file, info)
}
private fun MediaUploadInfo.postProcess(uri: Uri): MediaUploadInfo { private fun MediaUploadInfo.postProcess(uri: Uri): MediaUploadInfo {
val name = context.getFileName(uri) ?: return this val name = context.getFileName(uri) ?: return this
val renamedFile = File(context.cacheDir, name).also { val renamedFile = File(context.cacheDir, name).also {
@ -142,40 +114,83 @@ class AndroidMediaPreProcessor @Inject constructor(
} }
} }
private suspend fun processImage(uri: Uri): MediaUploadInfo { private suspend fun processImage(uri: Uri, mimeType: String, shouldBeCompressed: Boolean): MediaUploadInfo {
val compressedFileResult = contentResolver.openInputStream(uri).use { input ->
imageCompressor.compressToTmpFile( suspend fun processImageWithCompression(): MediaUploadInfo {
inputStream = requireNotNull(input), val compressionResult = contentResolver.openInputStream(uri).use { input ->
resizeMode = ResizeMode.Approximate(IMAGE_SCALE_REF_SIZE, IMAGE_SCALE_REF_SIZE), imageCompressor.compressToTmpFile(
).getOrThrow() inputStream = requireNotNull(input),
resizeMode = ResizeMode.Approximate(IMAGE_SCALE_REF_SIZE, IMAGE_SCALE_REF_SIZE),
).getOrThrow()
}
val thumbnailResult: ThumbnailResult = thumbnailFactory.createImageThumbnail(compressionResult.file)
val imageInfo = compressionResult.toImageInfo(
mimeType = mimeType,
thumbnailResult = thumbnailResult
)
removeSensitiveImageMetadata(compressionResult.file)
return MediaUploadInfo.Image(
file = compressionResult.file,
info = imageInfo,
thumbnailFile = thumbnailResult.file
)
} }
removeSensitiveImageMetadata(compressedFileResult.file) suspend fun processImageWithoutCompression(): MediaUploadInfo {
val file = copyToTmpFile(uri)
val thumbnailResult: ThumbnailResult = thumbnailFactory.createImageThumbnail(file)
val imageInfo = contentResolver.openInputStream(uri).use { input ->
val bitmap = BitmapFactory.decodeStream(input, null, null)!!
ImageInfo(
width = bitmap.width.toLong(),
height = bitmap.height.toLong(),
mimetype = mimeType,
size = file.length(),
thumbnailInfo = thumbnailResult.info,
thumbnailSource = null,
blurhash = thumbnailResult.blurhash,
)
}
removeSensitiveImageMetadata(file)
return MediaUploadInfo.Image(
file = file,
info = imageInfo,
thumbnailFile = thumbnailResult.file
)
}
val thumbnailResult = compressedFileResult.file.inputStream().use { generateImageThumbnail(it) } return if (shouldBeCompressed) {
val processingResult = compressedFileResult.toImageInfo(MimeTypes.Jpeg, thumbnailResult.file.path, thumbnailResult.info) processImageWithCompression()
return MediaUploadInfo.Image(compressedFileResult.file, processingResult, thumbnailResult) } else {
processImageWithoutCompression()
}
} }
private suspend fun processVideo(uri: Uri, mimeType: String?): MediaUploadInfo { private suspend fun processVideo(uri: Uri, mimeType: String?, shouldBeCompressed: Boolean): MediaUploadInfo {
val thumbnailInfo = extractVideoThumbnail(uri) val resultFile = if (shouldBeCompressed) {
val resultFile = videoCompressor.compress(uri) videoCompressor.compress(uri)
.onEach { .onEach {
// TODO handle progress // TODO handle progress
} }
.filterIsInstance<VideoTranscodingEvent.Completed>() .filterIsInstance<VideoTranscodingEvent.Completed>()
.first() .first()
.file .file
} else {
val videoProcessingInfo = extractVideoMetadata(resultFile, mimeType, thumbnailInfo.file.path, thumbnailInfo) copyToTmpFile(uri)
return MediaUploadInfo.Video(resultFile, videoProcessingInfo, thumbnailInfo) }
val thumbnailInfo = thumbnailFactory.createVideoThumbnail(resultFile)
val videoInfo = extractVideoMetadata(resultFile, mimeType, thumbnailInfo)
return MediaUploadInfo.Video(
file = resultFile,
info = videoInfo,
thumbnailFile = thumbnailInfo.file
)
} }
private suspend fun processAudio(uri: Uri, mimeType: String?): MediaUploadInfo { private suspend fun processAudio(uri: Uri, mimeType: String?): MediaUploadInfo {
val file = copyToTmpFile(uri) val file = copyToTmpFile(uri)
return MediaMetadataRetriever().runAndRelease { return MediaMetadataRetriever().runAndRelease {
setDataSource(context, Uri.fromFile(file)) setDataSource(context, Uri.fromFile(file))
val info = AudioInfo( val info = AudioInfo(
duration = extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)?.toLong() ?: 0L, duration = extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)?.toLong() ?: 0L,
size = file.length(), size = file.length(),
@ -186,15 +201,6 @@ class AndroidMediaPreProcessor @Inject constructor(
} }
} }
private suspend fun generateImageThumbnail(inputStream: InputStream): ThumbnailProcessingInfo {
val thumbnailResult = imageCompressor
.compressToTmpFile(
inputStream = inputStream,
resizeMode = ResizeMode.Strict(THUMB_MAX_WIDTH, THUMB_MAX_HEIGHT),
).getOrThrow()
return thumbnailResult.toThumbnailProcessingInfo(MimeTypes.Jpeg)
}
private fun removeSensitiveImageMetadata(file: File) { private fun removeSensitiveImageMetadata(file: File) {
// Remove GPS info, user comments and subject location tags // Remove GPS info, user comments and subject location tags
val exifInterface = ExifInterface(file) val exifInterface = ExifInterface(file)
@ -215,7 +221,7 @@ class AndroidMediaPreProcessor @Inject constructor(
} }
} }
private fun extractVideoMetadata(file: File, mimeType: String?, thumbnailUrl: String?, thumbnailInfo: ThumbnailProcessingInfo?): VideoInfo = private fun extractVideoMetadata(file: File, mimeType: String?, thumbnailResult: ThumbnailResult): VideoInfo =
MediaMetadataRetriever().runAndRelease { MediaMetadataRetriever().runAndRelease {
setDataSource(context, Uri.fromFile(file)) setDataSource(context, Uri.fromFile(file))
VideoInfo( VideoInfo(
@ -224,51 +230,26 @@ class AndroidMediaPreProcessor @Inject constructor(
height = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toLong() ?: 0L, height = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toLong() ?: 0L,
mimetype = mimeType, mimetype = mimeType,
size = file.length(), size = file.length(),
thumbnailInfo = thumbnailInfo?.info, thumbnailInfo = thumbnailResult.info,
thumbnailSource = thumbnailUrl?.let { MediaSource(it) }, // Will be computed by the rust sdk
blurhash = thumbnailInfo?.blurhash, thumbnailSource = null,
blurhash = thumbnailResult.blurhash,
) )
} }
private suspend fun extractVideoThumbnail(uri: Uri): ThumbnailProcessingInfo =
MediaMetadataRetriever().runAndRelease {
setDataSource(context, uri)
val bitmap = requireNotNull(getFrameAtTime(VIDEO_THUMB_FRAME))
val inputStream = ByteArrayOutputStream().use {
bitmap.compress(Bitmap.CompressFormat.JPEG, 80, it)
ByteArrayInputStream(it.toByteArray())
}
val result = imageCompressor.compressToTmpFile(
inputStream = inputStream,
resizeMode = ResizeMode.Strict(THUMB_MAX_WIDTH, THUMB_MAX_HEIGHT),
)
result.getOrThrow().toThumbnailProcessingInfo(MimeTypes.Jpeg)
}
private suspend fun copyToTmpFile(uri: Uri): File { private suspend fun copyToTmpFile(uri: Uri): File {
return contentResolver.openInputStream(uri)?.use { createTmpFileWithInput(it) } return contentResolver.openInputStream(uri)?.use { createTmpFileWithInput(it) }
?: error("Could not copy the contents of $uri to a temporary file") ?: error("Could not copy the contents of $uri to a temporary file")
} }
} }
fun ImageCompressionResult.toImageInfo(mimeType: String, thumbnailUrl: String?, thumbnailInfo: ThumbnailInfo?) = ImageInfo( fun ImageCompressionResult.toImageInfo(mimeType: String, thumbnailResult: ThumbnailResult) = ImageInfo(
width = width.toLong(), width = width.toLong(),
height = height.toLong(), height = height.toLong(),
mimetype = mimeType, mimetype = mimeType,
size = size, size = size,
thumbnailInfo = thumbnailInfo, thumbnailInfo = thumbnailResult.info,
thumbnailSource = thumbnailUrl?.let { MediaSource(it) }, // Will be computed by the rust sdk
blurhash = blurhash, thumbnailSource = null,
) blurhash = thumbnailResult.blurhash,
fun ImageCompressionResult.toThumbnailProcessingInfo(mimeType: String) = ThumbnailProcessingInfo(
file = file,
info = ThumbnailInfo(
width = width.toLong(),
height = height.toLong(),
mimetype = mimeType,
size = size,
),
blurhash = blurhash,
) )

View file

@ -42,27 +42,23 @@ class ImageCompressor @Inject constructor(
* @return a [Result] containing the resulting [ImageCompressionResult] with the temporary [File] and some metadata. * @return a [Result] containing the resulting [ImageCompressionResult] with the temporary [File] and some metadata.
*/ */
suspend fun compressToTmpFile( suspend fun compressToTmpFile(
inputStream: InputStream, inputStream: InputStream,
resizeMode: ResizeMode, resizeMode: ResizeMode,
format: Bitmap.CompressFormat = Bitmap.CompressFormat.JPEG, format: Bitmap.CompressFormat = Bitmap.CompressFormat.JPEG,
desiredQuality: Int = 80, desiredQuality: Int = 80,
): Result<ImageCompressionResult> = withContext(Dispatchers.IO) { ): Result<ImageCompressionResult> = withContext(Dispatchers.IO) {
runCatching { runCatching {
val compressedBitmap = compressToBitmap(inputStream, resizeMode).getOrThrow() val compressedBitmap = compressToBitmap(inputStream, resizeMode).getOrThrow()
val blurhash = BlurHash.encode(compressedBitmap, 3, 3)
// Encode bitmap to the destination temporary file // Encode bitmap to the destination temporary file
val tmpFile = context.createTmpFile(extension = "jpeg") val tmpFile = context.createTmpFile(extension = "jpeg")
tmpFile.outputStream().use { tmpFile.outputStream().use {
compressedBitmap.compress(format, desiredQuality, it) compressedBitmap.compress(format, desiredQuality, it)
} }
ImageCompressionResult( ImageCompressionResult(
file = tmpFile, file = tmpFile,
width = compressedBitmap.width, width = compressedBitmap.width,
height = compressedBitmap.height, height = compressedBitmap.height,
size = tmpFile.length(), size = tmpFile.length()
blurhash = blurhash
) )
} }
} }
@ -116,7 +112,6 @@ data class ImageCompressionResult(
val width: Int, val width: Int,
val height: Int, val height: Int,
val size: Long, val size: Long,
val blurhash: String,
) )
sealed interface ResizeMode { sealed interface ResizeMode {

View file

@ -0,0 +1,122 @@
/*
* Copyright (c) 2023 New Vector Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.element.android.libraries.mediaupload
import android.annotation.SuppressLint
import android.content.Context
import android.graphics.Bitmap
import android.media.MediaMetadataRetriever
import android.media.ThumbnailUtils
import android.os.Build
import android.os.CancellationSignal
import android.provider.MediaStore
import android.util.Size
import androidx.core.net.toUri
import com.vanniktech.blurhash.BlurHash
import io.element.android.libraries.androidutils.file.createTmpFile
import io.element.android.libraries.androidutils.media.runAndRelease
import io.element.android.libraries.core.mimetype.MimeTypes
import io.element.android.libraries.di.ApplicationContext
import io.element.android.libraries.matrix.api.media.ThumbnailInfo
import kotlinx.coroutines.suspendCancellableCoroutine
import java.io.File
import javax.inject.Inject
import kotlin.coroutines.resume
/**
* Max width of thumbnail images.
* See [the Matrix spec](https://spec.matrix.org/latest/client-server-api/?ref=blog.gitter.im#thumbnails).
*/
private const val THUMB_MAX_WIDTH = 800
/**
* Max height of thumbnail images.
* See [the Matrix spec](https://spec.matrix.org/latest/client-server-api/?ref=blog.gitter.im#thumbnails).
*/
private const val THUMB_MAX_HEIGHT = 600
/**
* Frame of the video to be used for generating a thumbnail.
*/
private const val VIDEO_THUMB_FRAME = 0L
class ThumbnailFactory @Inject constructor(
@ApplicationContext private val context: Context,
) {
@SuppressLint("NewApi")
suspend fun createImageThumbnail(file: File): ThumbnailResult {
return createThumbnail { cancellationSignal ->
// This API works correctly with GIF
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
ThumbnailUtils.createImageThumbnail(
file,
Size(THUMB_MAX_WIDTH, THUMB_MAX_HEIGHT),
cancellationSignal
)
} else {
ThumbnailUtils.createImageThumbnail(
file.path,
MediaStore.Images.Thumbnails.MINI_KIND,
)
}
}
}
suspend fun createVideoThumbnail(file: File): ThumbnailResult {
return createThumbnail {
MediaMetadataRetriever().runAndRelease {
setDataSource(context, file.toUri())
getFrameAtTime(VIDEO_THUMB_FRAME)
}
}
}
private suspend fun createThumbnail(bitmapThumbnailFactory: (CancellationSignal) -> Bitmap?): ThumbnailResult = suspendCancellableCoroutine { continuation ->
val cancellationSignal = CancellationSignal()
continuation.invokeOnCancellation {
cancellationSignal.cancel()
}
val bitmapThumbnail: Bitmap? = bitmapThumbnailFactory(cancellationSignal)
val thumbnailFile = context.createTmpFile(extension = "jpeg")
thumbnailFile.outputStream().use { outputStream ->
bitmapThumbnail?.compress(Bitmap.CompressFormat.JPEG, 80, outputStream)
}
val blurhash = bitmapThumbnail?.let {
BlurHash.encode(it, 3, 3)
}
val thumbnailResult = ThumbnailResult(
file = thumbnailFile,
info = ThumbnailInfo(
height = bitmapThumbnail?.height?.toLong(),
width = bitmapThumbnail?.width?.toLong(),
mimetype = MimeTypes.Jpeg,
size = thumbnailFile.length()
),
blurhash = blurhash
)
bitmapThumbnail?.recycle()
continuation.resume(thumbnailResult)
}
}
data class ThumbnailResult(
val file: File,
val info: ThumbnailInfo,
val blurhash: String?,
)