Sulkta-Coop/element-x-ada
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add waveform to voice message preview UI (#1661)

Browse source 
* Add waveform to preview UI

* Update screenshots

* Make random waveform function deterministic

* Update screenshots

---------

Co-authored-by: ElementBot <benoitm+elementbot@element.io>
This commit is contained in:
jonnyandrew 2023-10-27 12:33:35 +01:00 • committed by GitHub
parent 44de6adb86
commit 9510d43289
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 157 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

7
features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/WaveformUtils.kt
View file

@ -20,4 +20,9 @@ package io.element.android.features.messages.impl.voicemessages
* Resizes the given [0;1024] int list as per unstable MSC3246 spec
* to a [0;1] range float list to be used for waveform rendering.
*/
fun List<Int>.fromMSC3246range(): List<Float> = map { it / 1024f }
internal fun List<Int>.fromMSC3246range(): List<Float> = map { it / 1024f }
/**
* Resizes the given [0;1] float list to [0;1024] int list as per unstable MSC3246 spec.
*/
internal fun List<Float>.toMSC3246range(): List<Int> = map { (it * 1024).toInt() }

23
features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/composer/VoiceMessageComposerPresenter.kt
View file

@ -28,6 +28,7 @@ import androidx.compose.runtime.setValue
import androidx.core.net.toUri
import androidx.lifecycle.Lifecycle
import io.element.android.features.messages.impl.voicemessages.VoiceMessageException
import io.element.android.features.messages.impl.voicemessages.toMSC3246range
import io.element.android.libraries.architecture.Presenter
import io.element.android.libraries.di.RoomScope
import io.element.android.libraries.di.SingleIn
@ -40,6 +41,8 @@ import io.element.android.libraries.textcomposer.model.VoiceMessageState
import io.element.android.libraries.voicerecorder.api.VoiceRecorder
import io.element.android.libraries.voicerecorder.api.VoiceRecorderState
import io.element.android.services.analytics.api.AnalyticsService
import kotlinx.collections.immutable.ImmutableList
import kotlinx.collections.immutable.toImmutableList
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.launch
import timber.log.Timber
@ -66,6 +69,7 @@ class VoiceMessageComposerPresenter @Inject constructor(
var isSending by remember { mutableStateOf(false) }
val playerState by player.state.collectAsState(initial = VoiceMessageComposerPlayer.State.NotPlaying)
val isPlaying by remember(playerState.isPlaying) { derivedStateOf { playerState.isPlaying } }
val waveform by remember(recorderState) { derivedStateOf { recorderState.finishedWaveform() } }
val onLifecycleEvent = { event: Lifecycle.Event ->
when (event) {
@ -174,11 +178,11 @@ class VoiceMessageComposerPresenter @Inject constructor(
duration = state.elapsedTime,
level = state.level
)
is VoiceRecorderState.Finished -> if (isSending) {
VoiceMessageState.Sending
} else {
VoiceMessageState.Preview(isPlaying = isPlaying)
}
is VoiceRecorderState.Finished -> VoiceMessageState.Preview(
isSending = isSending,
isPlaying = isPlaying,
waveform = waveform,
)
else -> VoiceMessageState.Idle
},
showPermissionRationaleDialog = permissionState.showDialog,
@ -227,7 +231,8 @@ class VoiceMessageComposerPresenter @Inject constructor(
}
}
/**
* Resizes the given [0;1] float list to [0;1024] int list as per unstable MSC3246 spec.
*/
private fun List<Float>.toMSC3246range(): List<Int> = map { (it * 1024).toInt() }
private fun VoiceRecorderState.finishedWaveform(): ImmutableList<Float> =
(this as? VoiceRecorderState.Finished)
?.waveform
.orEmpty()
.toImmutableList()

43
features/messages/impl/src/test/kotlin/io/element/android/features/messages/voicemessages/composer/VoiceMessageComposerPresenterTest.kt
View file

@ -25,11 +25,11 @@ import app.cash.molecule.moleculeFlow
import app.cash.turbine.TurbineTestContext
import app.cash.turbine.test
import com.google.common.truth.Truth.assertThat
import io.element.android.features.messages.impl.voicemessages.VoiceMessageException
import io.element.android.features.messages.impl.voicemessages.composer.VoiceMessageComposerEvents
import io.element.android.features.messages.impl.voicemessages.composer.VoiceMessageComposerPlayer
import io.element.android.features.messages.impl.voicemessages.composer.VoiceMessageComposerPresenter
import io.element.android.features.messages.impl.voicemessages.composer.VoiceMessageComposerState
import io.element.android.features.messages.impl.voicemessages.VoiceMessageException
import io.element.android.features.messages.impl.voicemessages.composer.VoiceMessageComposerPlayer
import io.element.android.libraries.mediaplayer.test.FakeMediaPlayer
import io.element.android.libraries.matrix.test.room.FakeMatrixRoom
import io.element.android.libraries.mediaupload.api.MediaSender
@ -44,6 +44,7 @@ import io.element.android.libraries.textcomposer.model.VoiceMessageState
import io.element.android.libraries.voicerecorder.test.FakeVoiceRecorder
import io.element.android.services.analytics.test.FakeAnalyticsService
import io.element.android.tests.testutils.WarmUpRule
import kotlinx.collections.immutable.toImmutableList
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.test.TestScope
import kotlinx.coroutines.test.runTest
@ -216,7 +217,7 @@ class VoiceMessageComposerPresenterTest {
awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart))
awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.LongPressEnd))
awaitItem().eventSink(VoiceMessageComposerEvents.SendVoiceMessage)
assertThat(awaitItem().voiceMessageState).isEqualTo(VoiceMessageState.Sending)
assertThat(awaitItem().voiceMessageState).isEqualTo(aPreviewState(isSending = true))
val finalState = awaitItem()
assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Idle)
@ -239,7 +240,7 @@ class VoiceMessageComposerPresenterTest {
eventSink(VoiceMessageComposerEvents.SendVoiceMessage)
eventSink(VoiceMessageComposerEvents.SendVoiceMessage)
}
assertThat(awaitItem().voiceMessageState).isEqualTo(VoiceMessageState.Sending)
assertThat(awaitItem().voiceMessageState).isEqualTo(aPreviewState(isSending = true))
val finalState = awaitItem()
assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Idle)
@ -266,7 +267,7 @@ class VoiceMessageComposerPresenterTest {
}
val finalState = awaitItem()
assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Sending)
assertThat(finalState.voiceMessageState).isEqualTo(aPreviewState(isSending = true))
assertThat(matrixRoom.sendMediaCount).isEqualTo(0)
assertThat(analyticsService.trackedErrors).hasSize(0)
voiceRecorder.assertCalls(started = 1, stopped = 1, deleted = 0)
@ -288,7 +289,7 @@ class VoiceMessageComposerPresenterTest {
val previewState = awaitItem()
previewState.eventSink(VoiceMessageComposerEvents.SendVoiceMessage)
assertThat(awaitItem().voiceMessageState).isEqualTo(VoiceMessageState.Sending)
assertThat(awaitItem().voiceMessageState).isEqualTo(aPreviewState(isSending = true))
ensureAllEventsConsumed()
assertThat(previewState.voiceMessageState).isEqualTo(aPreviewState())
@ -452,18 +453,15 @@ class VoiceMessageComposerPresenterTest {
VoiceMessageComposerEvents.LifecycleEvent(event = Lifecycle.Event.ON_PAUSE)
)
val onPauseState = when (val vmState = mostRecentState.voiceMessageState) {
VoiceMessageState.Idle,
VoiceMessageState.Sending -> {
mostRecentState
}
val onPauseState = when (val state = mostRecentState.voiceMessageState) {
VoiceMessageState.Idle -> mostRecentState
is VoiceMessageState.Recording -> {
// If recorder was active, it stops
awaitItem().apply {
assertThat(voiceMessageState).isEqualTo(aPreviewState())
}
}
is VoiceMessageState.Preview -> when(vmState.isPlaying) {
is VoiceMessageState.Preview -> when (state.isPlaying) {
// If the preview was playing, it pauses
true -> awaitItem().apply {
assertThat(voiceMessageState).isEqualTo(aPreviewState())
@ -476,13 +474,15 @@ class VoiceMessageComposerPresenterTest {
VoiceMessageComposerEvents.LifecycleEvent(event = Lifecycle.Event.ON_DESTROY)
)
when (onPauseState.voiceMessageState) {
VoiceMessageState.Idle,
VoiceMessageState.Sending ->
when (val state = onPauseState.voiceMessageState) {
VoiceMessageState.Idle ->
ensureAllEventsConsumed()
is VoiceMessageState.Recording,
is VoiceMessageState.Preview ->
is VoiceMessageState.Recording ->
assertThat(awaitItem().voiceMessageState).isEqualTo(VoiceMessageState.Idle)
is VoiceMessageState.Preview -> when (state.isSending) {
true -> ensureAllEventsConsumed()
false -> assertThat(awaitItem().voiceMessageState).isEqualTo(VoiceMessageState.Idle)
}
}
}
@ -514,9 +514,12 @@ class VoiceMessageComposerPresenterTest {
}
private fun aPreviewState(
isPlaying: Boolean = false
isPlaying: Boolean = false,
isSending: Boolean = false,
waveform: List<Float> = voiceRecorder.waveform,
) = VoiceMessageState.Preview(
isPlaying = isPlaying
isPlaying = isPlaying,
isSending = isSending,
waveform = waveform.toImmutableList(),
)
}

36
libraries/designsystem/src/main/kotlin/io/element/android/libraries/designsystem/components/media/FakeWaveformFactory.kt Normal file
View file

@ -0,0 +1,36 @@
/*
* Copyright (c) 2023 New Vector Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.element.android.libraries.designsystem.components.media
import kotlinx.collections.immutable.ImmutableList
import kotlinx.collections.immutable.toPersistentList
import kotlin.random.Random
object FakeWaveformFactory {
/**
* Generate a waveform for testing purposes.
*
* The waveform is a list of floats between 0 and 1.
*
* @param length The length of the waveform.
*/
fun createFakeWaveform(length: Int = 1000): ImmutableList<Float> {
val random = Random(seed = 2)
return List(length) { random.nextFloat() }
.toPersistentList()
}
}

17
libraries/designsystem/src/main/kotlin/io/element/android/libraries/designsystem/components/media/WaveformPlaybackView.kt
View file

@ -54,6 +54,21 @@ import kotlin.math.roundToInt
private const val DEFAULT_GRAPHICS_LAYER_ALPHA: Float = 0.99F
/**
* A view that displays a waveform and a cursor to indicate the current playback progress.
*
* @param playbackProgress The current playback progress, between 0 and 1.
* @param showCursor Whether to show the cursor or not.
* @param waveform The waveform to display. Use [FakeWaveformFactory] to generate a fake waveform.
* @param modifier The modifier to be applied to the view.
* @param onSeek Callback when the user seeks the waveform. Called with a value between 0 and 1.
* @param brush The brush to use to draw the waveform.
* @param progressBrush The brush to use to draw the progress.
* @param cursorBrush The brush to use to draw the cursor.
* @param lineWidth The width of the waveform lines.
* @param linePadding The padding between waveform lines.
* @param minimumGraphAmplitude The minimum amplitude to display, regardless of waveform data.
*/
@OptIn(ExperimentalComposeUiApi::class)
@Composable
fun WaveformPlaybackView(
@ -78,7 +93,7 @@ fun WaveformPlaybackView(
}
}
val progressAnimated = animateFloatAsState(targetValue = progress, label = "progressAnimation")
val amplitudeDisplayCount by remember(canvasSize) {
val amplitudeDisplayCount by remember(canvasSize, lineWidth, linePadding) {
derivedStateOf {
(canvasSize.width.value / (lineWidth.value + linePadding.value)).toInt()
}

40
libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/TextComposer.kt
View file

@ -48,6 +48,7 @@ import androidx.compose.ui.res.stringResource
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import io.element.android.libraries.designsystem.components.media.FakeWaveformFactory.createFakeWaveform
import io.element.android.libraries.designsystem.preview.ElementPreview
import io.element.android.libraries.designsystem.preview.PreviewsDayNight
import io.element.android.libraries.designsystem.text.applyScaleUp
@ -122,6 +123,10 @@ fun TextComposer(
onVoicePlayerEvent(VoiceMessagePlayerEvent.Pause)
}
val onSeekVoiceMessage = { position: Float ->
onVoicePlayerEvent(VoiceMessagePlayerEvent.Seek(position))
}
val layoutModifier = modifier
.fillMaxSize()
.height(IntrinsicSize.Min)
@ -188,8 +193,10 @@ fun TextComposer(
when (voiceMessageState) {
VoiceMessageState.Idle,
is VoiceMessageState.Recording -> recordVoiceButton
is VoiceMessageState.Preview -> sendVoiceButton
is VoiceMessageState.Sending -> uploadVoiceProgress
is VoiceMessageState.Preview -> when(voiceMessageState.isSending) {
true -> uploadVoiceProgress
false -> sendVoiceButton
}
}
else ->
sendButton
@ -199,17 +206,12 @@ fun TextComposer(
when (voiceMessageState) {
is VoiceMessageState.Preview ->
VoiceMessagePreview(
isInteractive = true,
isInteractive = !voiceMessageState.isSending,
isPlaying = voiceMessageState.isPlaying,
waveform = voiceMessageState.waveform,
onPlayClick = onPlayVoiceMessageClicked,
onPauseClick = onPauseVoiceMessageClicked
)
VoiceMessageState.Sending ->
VoiceMessagePreview(
isInteractive = false,
isPlaying = false,
onPlayClick = onPlayVoiceMessageClicked,
onPauseClick = onPauseVoiceMessageClicked
onPauseClick = onPauseVoiceMessageClicked,
onSeek = onSeekVoiceMessage,
)
is VoiceMessageState.Recording ->
VoiceMessageRecording(voiceMessageState.level, voiceMessageState.duration)
@ -218,13 +220,9 @@ fun TextComposer(
}
val voiceDeleteButton = @Composable {
val enabled = when (voiceMessageState) {
is VoiceMessageState.Preview -> true
VoiceMessageState.Sending,
is VoiceMessageState.Recording,
VoiceMessageState.Idle -> false
if(voiceMessageState is VoiceMessageState.Preview) {
VoiceMessageDeleteButton(enabled = !voiceMessageState.isSending, onClick = onDeleteVoiceMessage)
}
VoiceMessageDeleteButton(enabled = enabled, onClick = onDeleteVoiceMessage)
}
if (showTextFormatting) {
@ -285,7 +283,7 @@ private fun StandardLayout(
verticalAlignment = Alignment.Bottom,
) {
if (enableVoiceMessages && voiceMessageState !is VoiceMessageState.Idle) {
if (voiceMessageState is VoiceMessageState.Preview || voiceMessageState is VoiceMessageState.Sending) {
if (voiceMessageState is VoiceMessageState.Preview) {
Box(
modifier = Modifier
.padding(bottom = 5.dp, top = 5.dp, end = 3.dp, start = 3.dp)
@ -818,11 +816,11 @@ internal fun TextComposerVoicePreview() = ElementPreview {
PreviewColumn(items = persistentListOf({
VoicePreview(voiceMessageState = VoiceMessageState.Recording(61.seconds, 0.5f))
}, {
VoicePreview(voiceMessageState = VoiceMessageState.Preview(isPlaying = false))
VoicePreview(voiceMessageState = VoiceMessageState.Preview(isSending = false, isPlaying = false, waveform = createFakeWaveform()))
}, {
VoicePreview(voiceMessageState = VoiceMessageState.Preview(isPlaying = true))
VoicePreview(voiceMessageState = VoiceMessageState.Preview(isSending = false, isPlaying = true, waveform = createFakeWaveform()))
}, {
VoicePreview(voiceMessageState = VoiceMessageState.Sending)
VoicePreview(voiceMessageState = VoiceMessageState.Preview(isSending = true, isPlaying = false, waveform = createFakeWaveform()))
}))
}

34
libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/VoiceMessagePreview.kt
View file

@ -20,10 +20,13 @@ import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.heightIn
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.material3.MaterialTheme
import androidx.compose.runtime.Composable
@ -31,6 +34,8 @@ import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.unit.dp
import io.element.android.libraries.designsystem.components.media.FakeWaveformFactory.createFakeWaveform
import io.element.android.libraries.designsystem.components.media.WaveformPlaybackView
import io.element.android.libraries.designsystem.preview.ElementPreview
import io.element.android.libraries.designsystem.preview.PreviewsDayNight
import io.element.android.libraries.designsystem.text.applyScaleUp
@ -39,14 +44,18 @@ import io.element.android.libraries.designsystem.theme.components.IconButton
import io.element.android.libraries.textcomposer.R
import io.element.android.libraries.theme.ElementTheme
import io.element.android.libraries.ui.strings.CommonStrings
import kotlinx.collections.immutable.ImmutableList
@Composable
internal fun VoiceMessagePreview(
isInteractive: Boolean,
isPlaying: Boolean,
waveform: ImmutableList<Float>,
modifier: Modifier = Modifier,
playbackProgress: Float = 0f,
onPlayClick: () -> Unit = {},
onPauseClick: () -> Unit = {}
onPauseClick: () -> Unit = {},
onSeek: (Float) -> Unit = {},
) {
Row(
modifier = modifier
@ -72,7 +81,22 @@ internal fun VoiceMessagePreview(
enabled = isInteractive
)
}
// TODO Add recording preview UI
Spacer(modifier = Modifier.width(8.dp))
// TODO Add timer UI
Spacer(modifier = Modifier.width(12.dp))
WaveformPlaybackView(
modifier = Modifier
.weight(1f)
.height(26.dp.applyScaleUp()),
playbackProgress = playbackProgress,
showCursor = isInteractive,
waveform = waveform,
onSeek = onSeek,
)
}
}
@ -119,8 +143,8 @@ internal fun VoiceMessagePreviewPreview() = ElementPreview {
Column(
verticalArrangement = Arrangement.spacedBy(8.dp)
) {
VoiceMessagePreview(isInteractive = true, isPlaying = true)
VoiceMessagePreview(isInteractive = true, isPlaying = false)
VoiceMessagePreview(isInteractive = false, isPlaying = false)
VoiceMessagePreview(isInteractive = true, isPlaying = true, waveform = createFakeWaveform())
VoiceMessagePreview(isInteractive = true, isPlaying = false, waveform = createFakeWaveform())
VoiceMessagePreview(isInteractive = false, isPlaying = false, waveform = createFakeWaveform())
}
}

5
libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/model/VoiceMessageState.kt
View file

@ -16,15 +16,18 @@
package io.element.android.libraries.textcomposer.model
import kotlinx.collections.immutable.ImmutableList
import kotlin.time.Duration
sealed class VoiceMessageState {
data object Idle: VoiceMessageState()
data class Preview(
val isSending: Boolean,
val isPlaying: Boolean,
val waveform: ImmutableList<Float>,
): VoiceMessageState()
data object Sending: VoiceMessageState()
data class Recording(
val duration: Duration,
val level: Float,

3
libraries/voicerecorder/test/src/main/kotlin/io/element/android/libraries/voicerecorder/test/FakeVoiceRecorder.kt
View file

@ -42,6 +42,7 @@ class FakeVoiceRecorder(
private var stoppedCount = 0
private var deletedCount = 0
var waveform: List<Float> = listOf(0f, 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f, 8f, 7f, 6f, 5f, 4f, 3f, 2f, 1f, 0f)
override suspend fun startRecord() {
startedCount += 1
val startedAt = timeSource.markNow()
@ -73,7 +74,7 @@ class FakeVoiceRecorder(
else -> VoiceRecorderState.Finished(
file = curRecording!!,
mimeType = "audio/ogg",
waveform = listOf(0f, 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f, 8f, 7f, 6f, 5f, 4f, 3f, 2f, 1f, 0f),
waveform = waveform,
)
}
)

4
tests/uitests/src/test/snapshots/images/ui_S_t[l.textcomposer.components_null_VoiceMessagePreview-D-15_15_null,NEXUS_5,1.0,en].png
View file

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:0ae485c12f93649418e661ed6f4a3f1393c3e1c703e8fce84e29ff9c4607ca7b
size 8699
oid sha256:ff4763716e5283a13f7028532cf22953f08d93512bd19ac98e774fd950173ad5
size 22534

4
tests/uitests/src/test/snapshots/images/ui_S_t[l.textcomposer.components_null_VoiceMessagePreview-N-15_16_null,NEXUS_5,1.0,en].png
View file

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2afaaa054ed17809447352d245a287854afee0d05c0efd7b00341daf733dc07b
size 6623
oid sha256:a88306846e0bf2958e235c5b0944376f4c4983ceb026b4da962deb10b249f315
size 19326

4
tests/uitests/src/test/snapshots/images/ui_S_t[l.textcomposer_null_TextComposerVoice-D-4_4_null,NEXUS_5,1.0,en].png
View file

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8eed5b8511637961df60be133688df984560cb84b333a9377e859446dfac2e04
size 18036
oid sha256:8fbbbac71323947df0f09b8614b23206add7dbdd36a629668f3474be8b89b5a8
size 25549

4
tests/uitests/src/test/snapshots/images/ui_S_t[l.textcomposer_null_TextComposerVoice-N-4_5_null,NEXUS_5,1.0,en].png
View file

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:14bcc3395316251b2e25a23f95b53030bf2e2ebe0623ef7e113330a5fd2f853d
size 15852
oid sha256:81863e643154025c515efb0070ceaad2e04b74517ddbd314d214d7d7dae2286d
size 23193