Merge pull request #617 from vector-im/feature/fga/fix_media_pre_processing
Feature/fga/fix media pre processing
This commit is contained in:
ganfra
GitHub
commit
c7b2ac5acd
25 changed files with 296 additions and 214 deletions
|
|
@ -24,22 +24,24 @@ import androidx.compose.runtime.Composable
|
|||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.unit.dp
|
||||
import kotlin.math.min
|
||||
|
||||
private const val MAX_HEIGHT_IN_DP = 360f
|
||||
private const val MIN_ASPECT_RATIO = 0.6f
|
||||
private const val MAX_ASPECT_RATIO = 4f
|
||||
private const val DEFAULT_ASPECT_RATIO = 1.33f
|
||||
|
||||
@Composable
|
||||
fun TimelineItemAspectRatioBox(
|
||||
height: Int?,
|
||||
aspectRatio: Float,
|
||||
aspectRatio: Float?,
|
||||
modifier: Modifier = Modifier,
|
||||
contentAlignment: Alignment = Alignment.TopStart,
|
||||
content: @Composable BoxScope.() -> Unit,
|
||||
content: @Composable (BoxScope.() -> Unit),
|
||||
) {
|
||||
// TODO should probably be moved to an ElementTheme.dimensions
|
||||
val maxHeight = min(300, height ?: 0)
|
||||
val safeAspectRatio = (aspectRatio ?: DEFAULT_ASPECT_RATIO).coerceIn(MIN_ASPECT_RATIO, MAX_ASPECT_RATIO)
|
||||
Box(
|
||||
modifier = modifier
|
||||
.heightIn(max = maxHeight.dp)
|
||||
.aspectRatio(aspectRatio, matchHeightConstraintsFirst = true),
|
||||
.heightIn(max = MAX_HEIGHT_IN_DP.dp)
|
||||
.aspectRatio(safeAspectRatio, true),
|
||||
contentAlignment = contentAlignment,
|
||||
content = content
|
||||
)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@
|
|||
|
||||
package io.element.android.features.messages.impl.timeline.components.event
|
||||
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.layout.ContentScale
|
||||
|
|
@ -28,25 +27,20 @@ import io.element.android.libraries.designsystem.components.BlurHashAsyncImage
|
|||
import io.element.android.libraries.designsystem.preview.ElementPreviewDark
|
||||
import io.element.android.libraries.designsystem.preview.ElementPreviewLight
|
||||
import io.element.android.libraries.matrix.ui.media.MediaRequestData
|
||||
import kotlin.math.max
|
||||
|
||||
@Composable
|
||||
fun TimelineItemImageView(
|
||||
content: TimelineItemImageContent,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
// TODO place this value somewhere else?
|
||||
val minHeight = max(100, content.height ?: 0)
|
||||
TimelineItemAspectRatioBox(
|
||||
height = minHeight,
|
||||
aspectRatio = content.aspectRatio,
|
||||
modifier = modifier
|
||||
) {
|
||||
BlurHashAsyncImage(
|
||||
model = MediaRequestData(content.preferredMediaSource, MediaRequestData.Kind.File(content.body, content.mimeType)),
|
||||
blurHash = content.blurhash,
|
||||
modifier = Modifier.fillMaxSize(),
|
||||
contentScale = ContentScale.Fit,
|
||||
contentScale = ContentScale.Crop,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ fun TimelineItemVideoView(
|
|||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
TimelineItemAspectRatioBox(
|
||||
height = content.height,
|
||||
aspectRatio = content.aspectRatio,
|
||||
modifier = modifier,
|
||||
contentAlignment = Alignment.Center,
|
||||
|
|
@ -51,8 +50,7 @@ fun TimelineItemVideoView(
|
|||
BlurHashAsyncImage(
|
||||
model = MediaRequestData(content.thumbnailSource, MediaRequestData.Kind.File(content.body, content.mimeType)),
|
||||
blurHash = content.blurHash,
|
||||
modifier = Modifier.fillMaxSize(),
|
||||
contentScale = ContentScale.Fit,
|
||||
contentScale = ContentScale.Crop,
|
||||
)
|
||||
Box(
|
||||
modifier = Modifier.roundedBackground(),
|
||||
|
|
|
|||
|
|
@ -102,11 +102,11 @@ class TimelineItemContentMessageFactory @Inject constructor(
|
|||
}
|
||||
}
|
||||
|
||||
private fun aspectRatioOf(width: Long?, height: Long?): Float {
|
||||
private fun aspectRatioOf(width: Long?, height: Long?): Float? {
|
||||
return if (height != null && width != null) {
|
||||
width.toFloat() / height.toFloat()
|
||||
} else {
|
||||
0.7f
|
||||
null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ data class TimelineItemImageContent(
|
|||
val blurhash: String?,
|
||||
val width: Int?,
|
||||
val height: Int?,
|
||||
val aspectRatio: Float
|
||||
val aspectRatio: Float?
|
||||
) : TimelineItemEventContent {
|
||||
override val type: String = "TimelineItemImageContent"
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ data class TimelineItemVideoContent(
|
|||
val duration: Long,
|
||||
val videoSource: MediaSource,
|
||||
val thumbnailSource: MediaSource?,
|
||||
val aspectRatio: Float,
|
||||
val aspectRatio: Float?,
|
||||
val blurHash: String?,
|
||||
val height: Int?,
|
||||
val width: Int?,
|
||||
|
|
|
|||
|
|
@ -36,7 +36,6 @@ import io.element.android.libraries.featureflag.api.FeatureFlagService
|
|||
import io.element.android.libraries.featureflag.api.FeatureFlags
|
||||
import io.element.android.libraries.featureflag.test.FakeFeatureFlagService
|
||||
import io.element.android.libraries.matrix.api.media.ImageInfo
|
||||
import io.element.android.libraries.matrix.api.media.ThumbnailInfo
|
||||
import io.element.android.libraries.matrix.api.media.VideoInfo
|
||||
import io.element.android.libraries.matrix.api.room.MatrixRoom
|
||||
import io.element.android.libraries.matrix.test.ANOTHER_MESSAGE
|
||||
|
|
@ -50,7 +49,6 @@ import io.element.android.libraries.mediapickers.test.FakePickerProvider
|
|||
import io.element.android.libraries.mediaupload.api.MediaPreProcessor
|
||||
import io.element.android.libraries.mediaupload.api.MediaSender
|
||||
import io.element.android.libraries.mediaupload.api.MediaUploadInfo
|
||||
import io.element.android.libraries.mediaupload.api.ThumbnailProcessingInfo
|
||||
import io.element.android.libraries.mediaupload.test.FakeMediaPreProcessor
|
||||
import io.element.android.libraries.textcomposer.MessageComposerMode
|
||||
import io.mockk.mockk
|
||||
|
|
@ -301,16 +299,7 @@ class MessageComposerPresenterTest {
|
|||
thumbnailSource = null,
|
||||
blurhash = null,
|
||||
),
|
||||
thumbnailInfo = ThumbnailProcessingInfo(
|
||||
file = File("/some/path"),
|
||||
info = ThumbnailInfo(
|
||||
width = null,
|
||||
height = null,
|
||||
mimetype = null,
|
||||
size = null,
|
||||
),
|
||||
blurhash = "",
|
||||
)
|
||||
thumbnailFile = File("/some/path")
|
||||
)
|
||||
)
|
||||
)
|
||||
|
|
@ -344,16 +333,7 @@ class MessageComposerPresenterTest {
|
|||
thumbnailSource = null,
|
||||
blurhash = null,
|
||||
),
|
||||
thumbnailInfo = ThumbnailProcessingInfo(
|
||||
file = File("/some/path"),
|
||||
info = ThumbnailInfo(
|
||||
width = null,
|
||||
height = null,
|
||||
mimetype = null,
|
||||
size = null,
|
||||
),
|
||||
blurhash = "",
|
||||
)
|
||||
thumbnailFile = File("/some/path")
|
||||
)
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -20,5 +20,9 @@ import android.media.MediaMetadataRetriever
|
|||
|
||||
/** [MediaMetadataRetriever] only implements `AutoClosable` since API 29, so we need to execute this to have the same in older APIs. */
|
||||
inline fun <T> MediaMetadataRetriever.runAndRelease(block: MediaMetadataRetriever.() -> T): T {
|
||||
return block().also { release() }
|
||||
return try {
|
||||
block()
|
||||
} finally {
|
||||
release()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,15 +44,26 @@ class MediaSender @Inject constructor(
|
|||
): Result<Unit> {
|
||||
return when (info) {
|
||||
is MediaUploadInfo.Image -> {
|
||||
sendImage(info.file, info.thumbnailInfo.file, info.info)
|
||||
sendImage(
|
||||
file = info.file,
|
||||
thumbnailFile = info.thumbnailFile,
|
||||
imageInfo = info.info
|
||||
)
|
||||
}
|
||||
|
||||
is MediaUploadInfo.Video -> {
|
||||
sendVideo(info.file, info.thumbnailInfo.file, info.info)
|
||||
sendVideo(
|
||||
file = info.file,
|
||||
thumbnailFile = info.thumbnailFile,
|
||||
videoInfo = info.info
|
||||
)
|
||||
}
|
||||
|
||||
is MediaUploadInfo.AnyFile -> {
|
||||
sendFile(info.file, info.info)
|
||||
sendFile(
|
||||
file = info.file,
|
||||
fileInfo = info.info
|
||||
)
|
||||
}
|
||||
else -> Result.failure(IllegalStateException("Unexpected MediaUploadInfo format: $info"))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ package io.element.android.libraries.mediaupload.api
|
|||
import io.element.android.libraries.matrix.api.media.AudioInfo
|
||||
import io.element.android.libraries.matrix.api.media.FileInfo
|
||||
import io.element.android.libraries.matrix.api.media.ImageInfo
|
||||
import io.element.android.libraries.matrix.api.media.ThumbnailInfo
|
||||
import io.element.android.libraries.matrix.api.media.VideoInfo
|
||||
import java.io.File
|
||||
|
||||
|
|
@ -27,14 +26,8 @@ sealed interface MediaUploadInfo {
|
|||
|
||||
val file: File
|
||||
|
||||
data class Image(override val file: File, val info: ImageInfo, val thumbnailInfo: ThumbnailProcessingInfo) : MediaUploadInfo
|
||||
data class Video(override val file: File, val info: VideoInfo, val thumbnailInfo: ThumbnailProcessingInfo) : MediaUploadInfo
|
||||
data class Image(override val file: File, val info: ImageInfo, val thumbnailFile: File) : MediaUploadInfo
|
||||
data class Video(override val file: File, val info: VideoInfo, val thumbnailFile: File) : MediaUploadInfo
|
||||
data class Audio(override val file: File, val info: AudioInfo) : MediaUploadInfo
|
||||
data class AnyFile(override val file: File, val info: FileInfo) : MediaUploadInfo
|
||||
}
|
||||
|
||||
data class ThumbnailProcessingInfo(
|
||||
val file: File,
|
||||
val info: ThumbnailInfo,
|
||||
val blurhash: String,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
package io.element.android.libraries.mediaupload
|
||||
|
||||
import android.content.Context
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.BitmapFactory
|
||||
import android.media.MediaMetadataRetriever
|
||||
import android.net.Uri
|
||||
import androidx.exifinterface.media.ExifInterface
|
||||
|
|
@ -37,27 +37,22 @@ import io.element.android.libraries.di.ApplicationContext
|
|||
import io.element.android.libraries.matrix.api.media.AudioInfo
|
||||
import io.element.android.libraries.matrix.api.media.FileInfo
|
||||
import io.element.android.libraries.matrix.api.media.ImageInfo
|
||||
import io.element.android.libraries.matrix.api.media.MediaSource
|
||||
import io.element.android.libraries.matrix.api.media.ThumbnailInfo
|
||||
import io.element.android.libraries.matrix.api.media.VideoInfo
|
||||
import io.element.android.libraries.mediaupload.api.MediaPreProcessor
|
||||
import io.element.android.libraries.mediaupload.api.MediaUploadInfo
|
||||
import io.element.android.libraries.mediaupload.api.ThumbnailProcessingInfo
|
||||
import kotlinx.coroutines.flow.filterIsInstance
|
||||
import kotlinx.coroutines.flow.first
|
||||
import kotlinx.coroutines.flow.onEach
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.ByteArrayInputStream
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.io.File
|
||||
import java.io.InputStream
|
||||
import java.time.Duration
|
||||
import javax.inject.Inject
|
||||
import kotlin.time.Duration.Companion.seconds
|
||||
|
||||
@ContributesBinding(AppScope::class)
|
||||
class AndroidMediaPreProcessor @Inject constructor(
|
||||
@ApplicationContext private val context: Context,
|
||||
private val thumbnailFactory: ThumbnailFactory,
|
||||
private val imageCompressor: ImageCompressor,
|
||||
private val videoCompressor: VideoCompressor,
|
||||
private val coroutineDispatchers: CoroutineDispatchers,
|
||||
|
|
@ -70,23 +65,6 @@ class AndroidMediaPreProcessor @Inject constructor(
|
|||
* values may surpass this limit. (i.e.: an image of `480x3000px` would have `inSampleSize=1` and be sent as is).
|
||||
*/
|
||||
private const val IMAGE_SCALE_REF_SIZE = 640
|
||||
|
||||
/**
|
||||
* Max width of thumbnail images.
|
||||
* See [the Matrix spec](https://spec.matrix.org/latest/client-server-api/?ref=blog.gitter.im#thumbnails).
|
||||
*/
|
||||
private const val THUMB_MAX_WIDTH = 800
|
||||
|
||||
/**
|
||||
* Max height of thumbnail images.
|
||||
* See [the Matrix spec](https://spec.matrix.org/latest/client-server-api/?ref=blog.gitter.im#thumbnails).
|
||||
*/
|
||||
private const val THUMB_MAX_HEIGHT = 600
|
||||
|
||||
/**
|
||||
* Frame of the video to be used for generating a thumbnail.
|
||||
*/
|
||||
private val VIDEO_THUMB_FRAME = 5.seconds.inWholeMicroseconds
|
||||
}
|
||||
|
||||
private val contentResolver = context.contentResolver
|
||||
|
|
@ -96,40 +74,34 @@ class AndroidMediaPreProcessor @Inject constructor(
|
|||
mimeType: String,
|
||||
deleteOriginal: Boolean,
|
||||
compressIfPossible: Boolean,
|
||||
): Result<MediaUploadInfo> = runCatching {
|
||||
val shouldBeCompressed = compressIfPossible &&
|
||||
(mimeType.isMimeTypeImage() && mimeType != MimeTypes.Gif) ||
|
||||
mimeType.isMimeTypeVideo()
|
||||
|
||||
val result = if (shouldBeCompressed) {
|
||||
when {
|
||||
mimeType.isMimeTypeImage() -> processImage(uri)
|
||||
mimeType.isMimeTypeVideo() -> processVideo(uri, mimeType)
|
||||
): Result<MediaUploadInfo> = withContext(coroutineDispatchers.computation) {
|
||||
runCatching {
|
||||
val result = when {
|
||||
mimeType.isMimeTypeImage() -> processImage(uri, mimeType, compressIfPossible && mimeType != MimeTypes.Gif)
|
||||
mimeType.isMimeTypeVideo() -> processVideo(uri, mimeType, compressIfPossible)
|
||||
mimeType.isMimeTypeAudio() -> processAudio(uri, mimeType)
|
||||
else -> error("Cannot compress file of type: $mimeType")
|
||||
else -> processFile(uri, mimeType)
|
||||
}
|
||||
} else {
|
||||
val file = copyToTmpFile(uri)
|
||||
// Remove image metadata here too
|
||||
if (mimeType.isMimeTypeImage() && mimeType != MimeTypes.Gif) {
|
||||
removeSensitiveImageMetadata(file)
|
||||
if (deleteOriginal) {
|
||||
tryOrNull {
|
||||
contentResolver.delete(uri, null, null)
|
||||
}
|
||||
}
|
||||
val info = FileInfo(
|
||||
mimetype = mimeType,
|
||||
size = file.length(),
|
||||
thumbnailInfo = null,
|
||||
thumbnailSource = null,
|
||||
)
|
||||
MediaUploadInfo.AnyFile(file, info)
|
||||
result.postProcess(uri)
|
||||
}
|
||||
if (deleteOriginal) {
|
||||
tryOrNull {
|
||||
contentResolver.delete(uri, null, null)
|
||||
}
|
||||
}
|
||||
result.postProcess(uri)
|
||||
}.mapFailure { MediaPreProcessor.Failure(it) }
|
||||
|
||||
private suspend fun processFile(uri: Uri, mimeType: String): MediaUploadInfo {
|
||||
val file = copyToTmpFile(uri)
|
||||
val info = FileInfo(
|
||||
mimetype = mimeType,
|
||||
size = file.length(),
|
||||
thumbnailInfo = null,
|
||||
thumbnailSource = null,
|
||||
)
|
||||
return MediaUploadInfo.AnyFile(file, info)
|
||||
}
|
||||
|
||||
private fun MediaUploadInfo.postProcess(uri: Uri): MediaUploadInfo {
|
||||
val name = context.getFileName(uri) ?: return this
|
||||
val renamedFile = File(context.cacheDir, name).also {
|
||||
|
|
@ -143,33 +115,77 @@ class AndroidMediaPreProcessor @Inject constructor(
|
|||
}
|
||||
}
|
||||
|
||||
private suspend fun processImage(uri: Uri): MediaUploadInfo {
|
||||
val compressedFileResult = contentResolver.openInputStream(uri).use { input ->
|
||||
imageCompressor.compressToTmpFile(
|
||||
inputStream = requireNotNull(input),
|
||||
resizeMode = ResizeMode.Approximate(IMAGE_SCALE_REF_SIZE, IMAGE_SCALE_REF_SIZE),
|
||||
).getOrThrow()
|
||||
private suspend fun processImage(uri: Uri, mimeType: String, shouldBeCompressed: Boolean): MediaUploadInfo {
|
||||
|
||||
suspend fun processImageWithCompression(): MediaUploadInfo {
|
||||
val compressionResult = contentResolver.openInputStream(uri).use { input ->
|
||||
imageCompressor.compressToTmpFile(
|
||||
inputStream = requireNotNull(input),
|
||||
resizeMode = ResizeMode.Approximate(IMAGE_SCALE_REF_SIZE, IMAGE_SCALE_REF_SIZE),
|
||||
).getOrThrow()
|
||||
}
|
||||
val thumbnailResult: ThumbnailResult = thumbnailFactory.createImageThumbnail(compressionResult.file)
|
||||
val imageInfo = compressionResult.toImageInfo(
|
||||
mimeType = mimeType,
|
||||
thumbnailResult = thumbnailResult
|
||||
)
|
||||
removeSensitiveImageMetadata(compressionResult.file)
|
||||
return MediaUploadInfo.Image(
|
||||
file = compressionResult.file,
|
||||
info = imageInfo,
|
||||
thumbnailFile = thumbnailResult.file
|
||||
)
|
||||
}
|
||||
|
||||
removeSensitiveImageMetadata(compressedFileResult.file)
|
||||
suspend fun processImageWithoutCompression(): MediaUploadInfo {
|
||||
val file = copyToTmpFile(uri)
|
||||
val thumbnailResult: ThumbnailResult = thumbnailFactory.createImageThumbnail(file)
|
||||
val imageInfo = contentResolver.openInputStream(uri).use { input ->
|
||||
val bitmap = BitmapFactory.decodeStream(input, null, null)!!
|
||||
ImageInfo(
|
||||
width = bitmap.width.toLong(),
|
||||
height = bitmap.height.toLong(),
|
||||
mimetype = mimeType,
|
||||
size = file.length(),
|
||||
thumbnailInfo = thumbnailResult.info,
|
||||
thumbnailSource = null,
|
||||
blurhash = thumbnailResult.blurhash,
|
||||
)
|
||||
}
|
||||
removeSensitiveImageMetadata(file)
|
||||
return MediaUploadInfo.Image(
|
||||
file = file,
|
||||
info = imageInfo,
|
||||
thumbnailFile = thumbnailResult.file
|
||||
)
|
||||
}
|
||||
|
||||
val thumbnailResult = compressedFileResult.file.inputStream().use { generateImageThumbnail(it) }
|
||||
val processingResult = compressedFileResult.toImageInfo(MimeTypes.Jpeg, thumbnailResult.file.path, thumbnailResult.info)
|
||||
return MediaUploadInfo.Image(compressedFileResult.file, processingResult, thumbnailResult)
|
||||
return if (shouldBeCompressed) {
|
||||
processImageWithCompression()
|
||||
} else {
|
||||
processImageWithoutCompression()
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun processVideo(uri: Uri, mimeType: String?): MediaUploadInfo {
|
||||
val thumbnailInfo = extractVideoThumbnail(uri)
|
||||
val resultFile = videoCompressor.compress(uri)
|
||||
.onEach {
|
||||
// TODO handle progress
|
||||
}
|
||||
.filterIsInstance<VideoTranscodingEvent.Completed>()
|
||||
.first()
|
||||
.file
|
||||
|
||||
val videoProcessingInfo = extractVideoMetadata(resultFile, mimeType, thumbnailInfo.file.path, thumbnailInfo)
|
||||
return MediaUploadInfo.Video(resultFile, videoProcessingInfo, thumbnailInfo)
|
||||
private suspend fun processVideo(uri: Uri, mimeType: String?, shouldBeCompressed: Boolean): MediaUploadInfo {
|
||||
val resultFile = if (shouldBeCompressed) {
|
||||
videoCompressor.compress(uri)
|
||||
.onEach {
|
||||
// TODO handle progress
|
||||
}
|
||||
.filterIsInstance<VideoTranscodingEvent.Completed>()
|
||||
.first()
|
||||
.file
|
||||
} else {
|
||||
copyToTmpFile(uri)
|
||||
}
|
||||
val thumbnailInfo = thumbnailFactory.createVideoThumbnail(resultFile)
|
||||
val videoInfo = extractVideoMetadata(resultFile, mimeType, thumbnailInfo)
|
||||
return MediaUploadInfo.Video(
|
||||
file = resultFile,
|
||||
info = videoInfo,
|
||||
thumbnailFile = thumbnailInfo.file
|
||||
)
|
||||
}
|
||||
|
||||
private suspend fun processAudio(uri: Uri, mimeType: String?): MediaUploadInfo {
|
||||
|
|
@ -186,15 +202,6 @@ class AndroidMediaPreProcessor @Inject constructor(
|
|||
}
|
||||
}
|
||||
|
||||
private suspend fun generateImageThumbnail(inputStream: InputStream): ThumbnailProcessingInfo {
|
||||
val thumbnailResult = imageCompressor
|
||||
.compressToTmpFile(
|
||||
inputStream = inputStream,
|
||||
resizeMode = ResizeMode.Strict(THUMB_MAX_WIDTH, THUMB_MAX_HEIGHT),
|
||||
).getOrThrow()
|
||||
return thumbnailResult.toThumbnailProcessingInfo(MimeTypes.Jpeg)
|
||||
}
|
||||
|
||||
private fun removeSensitiveImageMetadata(file: File) {
|
||||
// Remove GPS info, user comments and subject location tags
|
||||
val exifInterface = ExifInterface(file)
|
||||
|
|
@ -215,7 +222,7 @@ class AndroidMediaPreProcessor @Inject constructor(
|
|||
}
|
||||
}
|
||||
|
||||
private fun extractVideoMetadata(file: File, mimeType: String?, thumbnailUrl: String?, thumbnailInfo: ThumbnailProcessingInfo?): VideoInfo =
|
||||
private fun extractVideoMetadata(file: File, mimeType: String?, thumbnailResult: ThumbnailResult): VideoInfo =
|
||||
MediaMetadataRetriever().runAndRelease {
|
||||
setDataSource(context, Uri.fromFile(file))
|
||||
VideoInfo(
|
||||
|
|
@ -224,56 +231,32 @@ class AndroidMediaPreProcessor @Inject constructor(
|
|||
height = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toLong() ?: 0L,
|
||||
mimetype = mimeType,
|
||||
size = file.length(),
|
||||
thumbnailInfo = thumbnailInfo?.info,
|
||||
thumbnailSource = thumbnailUrl?.let { MediaSource(it) },
|
||||
blurhash = thumbnailInfo?.blurhash,
|
||||
thumbnailInfo = thumbnailResult.info,
|
||||
// Will be computed by the rust sdk
|
||||
thumbnailSource = null,
|
||||
blurhash = thumbnailResult.blurhash,
|
||||
)
|
||||
}
|
||||
|
||||
private suspend fun extractVideoThumbnail(uri: Uri): ThumbnailProcessingInfo =
|
||||
MediaMetadataRetriever().runAndRelease {
|
||||
setDataSource(context, uri)
|
||||
val bitmap = requireNotNull(getFrameAtTime(VIDEO_THUMB_FRAME))
|
||||
val inputStream = ByteArrayOutputStream().use {
|
||||
bitmap.compress(Bitmap.CompressFormat.JPEG, 80, it)
|
||||
ByteArrayInputStream(it.toByteArray())
|
||||
}
|
||||
|
||||
val result = imageCompressor.compressToTmpFile(
|
||||
inputStream = inputStream,
|
||||
resizeMode = ResizeMode.Strict(THUMB_MAX_WIDTH, THUMB_MAX_HEIGHT),
|
||||
)
|
||||
result.getOrThrow().toThumbnailProcessingInfo(MimeTypes.Jpeg)
|
||||
}
|
||||
|
||||
private suspend fun copyToTmpFile(uri: Uri): File {
|
||||
return contentResolver.openInputStream(uri)?.use { createTmpFileWithInput(it) }
|
||||
?: error("Could not copy the contents of $uri to a temporary file")
|
||||
}
|
||||
}
|
||||
|
||||
fun ImageCompressionResult.toImageInfo(mimeType: String, thumbnailResult: ThumbnailResult) = ImageInfo(
|
||||
width = width.toLong(),
|
||||
height = height.toLong(),
|
||||
mimetype = mimeType,
|
||||
size = size,
|
||||
thumbnailInfo = thumbnailResult.info,
|
||||
// Will be computed by the rust sdk
|
||||
thumbnailSource = null,
|
||||
blurhash = thumbnailResult.blurhash,
|
||||
)
|
||||
|
||||
private fun MediaMetadataRetriever.extractDuration(): Duration {
|
||||
val durationInMs = extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)?.toLong() ?: 0L
|
||||
return Duration.ofMillis(durationInMs)
|
||||
}
|
||||
|
||||
fun ImageCompressionResult.toImageInfo(mimeType: String, thumbnailUrl: String?, thumbnailInfo: ThumbnailInfo?) = ImageInfo(
|
||||
width = width.toLong(),
|
||||
height = height.toLong(),
|
||||
mimetype = mimeType,
|
||||
size = size,
|
||||
thumbnailInfo = thumbnailInfo,
|
||||
thumbnailSource = thumbnailUrl?.let { MediaSource(it) },
|
||||
blurhash = blurhash,
|
||||
)
|
||||
|
||||
fun ImageCompressionResult.toThumbnailProcessingInfo(mimeType: String) = ThumbnailProcessingInfo(
|
||||
file = file,
|
||||
info = ThumbnailInfo(
|
||||
width = width.toLong(),
|
||||
height = height.toLong(),
|
||||
mimetype = mimeType,
|
||||
size = size,
|
||||
),
|
||||
blurhash = blurhash,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -42,27 +42,23 @@ class ImageCompressor @Inject constructor(
|
|||
* @return a [Result] containing the resulting [ImageCompressionResult] with the temporary [File] and some metadata.
|
||||
*/
|
||||
suspend fun compressToTmpFile(
|
||||
inputStream: InputStream,
|
||||
resizeMode: ResizeMode,
|
||||
format: Bitmap.CompressFormat = Bitmap.CompressFormat.JPEG,
|
||||
desiredQuality: Int = 80,
|
||||
inputStream: InputStream,
|
||||
resizeMode: ResizeMode,
|
||||
format: Bitmap.CompressFormat = Bitmap.CompressFormat.JPEG,
|
||||
desiredQuality: Int = 80,
|
||||
): Result<ImageCompressionResult> = withContext(Dispatchers.IO) {
|
||||
runCatching {
|
||||
val compressedBitmap = compressToBitmap(inputStream, resizeMode).getOrThrow()
|
||||
val blurhash = BlurHash.encode(compressedBitmap, 3, 3)
|
||||
|
||||
// Encode bitmap to the destination temporary file
|
||||
val tmpFile = context.createTmpFile(extension = "jpeg")
|
||||
tmpFile.outputStream().use {
|
||||
compressedBitmap.compress(format, desiredQuality, it)
|
||||
}
|
||||
|
||||
ImageCompressionResult(
|
||||
file = tmpFile,
|
||||
width = compressedBitmap.width,
|
||||
height = compressedBitmap.height,
|
||||
size = tmpFile.length(),
|
||||
blurhash = blurhash
|
||||
size = tmpFile.length()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -116,7 +112,6 @@ data class ImageCompressionResult(
|
|||
val width: Int,
|
||||
val height: Int,
|
||||
val size: Long,
|
||||
val blurhash: String,
|
||||
)
|
||||
|
||||
sealed interface ResizeMode {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,122 @@
|
|||
/*
|
||||
* Copyright (c) 2023 New Vector Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.element.android.libraries.mediaupload
|
||||
|
||||
import android.annotation.SuppressLint
|
||||
import android.content.Context
|
||||
import android.graphics.Bitmap
|
||||
import android.media.MediaMetadataRetriever
|
||||
import android.media.ThumbnailUtils
|
||||
import android.os.Build
|
||||
import android.os.CancellationSignal
|
||||
import android.provider.MediaStore
|
||||
import android.util.Size
|
||||
import androidx.core.net.toUri
|
||||
import com.vanniktech.blurhash.BlurHash
|
||||
import io.element.android.libraries.androidutils.file.createTmpFile
|
||||
import io.element.android.libraries.androidutils.media.runAndRelease
|
||||
import io.element.android.libraries.core.mimetype.MimeTypes
|
||||
import io.element.android.libraries.di.ApplicationContext
|
||||
import io.element.android.libraries.matrix.api.media.ThumbnailInfo
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import java.io.File
|
||||
import javax.inject.Inject
|
||||
import kotlin.coroutines.resume
|
||||
|
||||
/**
|
||||
* Max width of thumbnail images.
|
||||
* See [the Matrix spec](https://spec.matrix.org/latest/client-server-api/?ref=blog.gitter.im#thumbnails).
|
||||
*/
|
||||
private const val THUMB_MAX_WIDTH = 800
|
||||
|
||||
/**
|
||||
* Max height of thumbnail images.
|
||||
* See [the Matrix spec](https://spec.matrix.org/latest/client-server-api/?ref=blog.gitter.im#thumbnails).
|
||||
*/
|
||||
private const val THUMB_MAX_HEIGHT = 600
|
||||
|
||||
/**
|
||||
* Frame of the video to be used for generating a thumbnail.
|
||||
*/
|
||||
private const val VIDEO_THUMB_FRAME = 0L
|
||||
|
||||
class ThumbnailFactory @Inject constructor(
|
||||
@ApplicationContext private val context: Context,
|
||||
) {
|
||||
|
||||
@SuppressLint("NewApi")
|
||||
suspend fun createImageThumbnail(file: File): ThumbnailResult {
|
||||
return createThumbnail { cancellationSignal ->
|
||||
// This API works correctly with GIF
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
ThumbnailUtils.createImageThumbnail(
|
||||
file,
|
||||
Size(THUMB_MAX_WIDTH, THUMB_MAX_HEIGHT),
|
||||
cancellationSignal
|
||||
)
|
||||
} else {
|
||||
ThumbnailUtils.createImageThumbnail(
|
||||
file.path,
|
||||
MediaStore.Images.Thumbnails.MINI_KIND,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun createVideoThumbnail(file: File): ThumbnailResult {
|
||||
return createThumbnail {
|
||||
MediaMetadataRetriever().runAndRelease {
|
||||
setDataSource(context, file.toUri())
|
||||
getFrameAtTime(VIDEO_THUMB_FRAME)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun createThumbnail(bitmapFactory: (CancellationSignal) -> Bitmap?): ThumbnailResult = suspendCancellableCoroutine { continuation ->
|
||||
val cancellationSignal = CancellationSignal()
|
||||
continuation.invokeOnCancellation {
|
||||
cancellationSignal.cancel()
|
||||
}
|
||||
val bitmapThumbnail: Bitmap? = bitmapFactory(cancellationSignal)
|
||||
val thumbnailFile = context.createTmpFile(extension = "jpeg")
|
||||
thumbnailFile.outputStream().use { outputStream ->
|
||||
bitmapThumbnail?.compress(Bitmap.CompressFormat.JPEG, 80, outputStream)
|
||||
}
|
||||
val blurhash = bitmapThumbnail?.let {
|
||||
BlurHash.encode(it, 3, 3)
|
||||
}
|
||||
val thumbnailResult = ThumbnailResult(
|
||||
file = thumbnailFile,
|
||||
info = ThumbnailInfo(
|
||||
height = bitmapThumbnail?.height?.toLong(),
|
||||
width = bitmapThumbnail?.width?.toLong(),
|
||||
mimetype = MimeTypes.Jpeg,
|
||||
size = thumbnailFile.length()
|
||||
),
|
||||
blurhash = blurhash
|
||||
)
|
||||
bitmapThumbnail?.recycle()
|
||||
continuation.resume(thumbnailResult)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
data class ThumbnailResult(
|
||||
val file: File,
|
||||
val info: ThumbnailInfo,
|
||||
val blurhash: String?,
|
||||
)
|
||||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fa17dd70b3c5eaaa37fbc5eed53997dd627090d9a5a3ffa518b9688647449a8e
|
||||
size 99065
|
||||
oid sha256:c40273c36eb1479f284e75fa91d4a75b1ae97edd0242dda37a2d4a8f10394928
|
||||
size 138700
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:4593d265265cea31c20b545250d20df54dfcc877d6e54eae28d1e84a1c693f16
|
||||
size 147443
|
||||
oid sha256:d7c47c713c74766c39d3b349d9e421ea588261e043105a5475cd8e68af782806
|
||||
size 185325
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fa17dd70b3c5eaaa37fbc5eed53997dd627090d9a5a3ffa518b9688647449a8e
|
||||
size 99065
|
||||
oid sha256:c40273c36eb1479f284e75fa91d4a75b1ae97edd0242dda37a2d4a8f10394928
|
||||
size 138700
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:4593d265265cea31c20b545250d20df54dfcc877d6e54eae28d1e84a1c693f16
|
||||
size 147443
|
||||
oid sha256:d7c47c713c74766c39d3b349d9e421ea588261e043105a5475cd8e68af782806
|
||||
size 185325
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f4e329e21d49bd79b633913edbbc1d5c63024874d866d96d399b1a8ffa5c1f18
|
||||
size 99209
|
||||
oid sha256:06617cda0f93ce0ea26b2a77a243371cdaf4f9a2956d8159ecb7196f7f6fe082
|
||||
size 139282
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5eb1c967db078a0333ef3fe26f94692264d0158d74d78768df32cc7c641faee3
|
||||
size 147537
|
||||
oid sha256:55d2d11ba729a68b1b62e9a69c3308594839fdb3612ff3db59850f0b346c28da
|
||||
size 186118
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f4e329e21d49bd79b633913edbbc1d5c63024874d866d96d399b1a8ffa5c1f18
|
||||
size 99209
|
||||
oid sha256:06617cda0f93ce0ea26b2a77a243371cdaf4f9a2956d8159ecb7196f7f6fe082
|
||||
size 139282
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5eb1c967db078a0333ef3fe26f94692264d0158d74d78768df32cc7c641faee3
|
||||
size 147537
|
||||
oid sha256:55d2d11ba729a68b1b62e9a69c3308594839fdb3612ff3db59850f0b346c28da
|
||||
size 186118
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ed788e9f85e922a81e077c2ad82b89fa822d9dec6d95442ad5aa5365535a87b9
|
||||
size 193815
|
||||
oid sha256:ead933d9b666f988a1c30997919ab364dacc4602c0cc62c03b14c63bd1b978d7
|
||||
size 226100
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:169f1a6dd0d433a900c43867a3c2feacd3143fc8e58029d4f24eeb4e5a41d2f2
|
||||
size 193901
|
||||
oid sha256:2d753eeb5d8e83643f858e5190cde015321e32959111f6a45e70a60f790e94c8
|
||||
size 226976
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c8943194bce27da10d2f047e5617cbb45548170cc1f8c9606e62dae20fde44f5
|
||||
size 194104
|
||||
oid sha256:7647550e41280854b7dd68110250d8f217acc8bef70a25ca0ebdf28994802d63
|
||||
size 225809
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:dc4c738cf6e3c8961d0e97b8adc24d0ba8765cd8a8d1799b9783a5b1f098fc25
|
||||
size 194152
|
||||
oid sha256:35150cf00f6e4d899417ff5f456e8fb9168424a75def962f4df40155d3819104
|
||||
size 226734
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue