Merge pull request #1659 from vector-im/langleyd/live_waveform

Live waveform
This commit is contained in:
David Langley 2023-10-27 14:41:16 +01:00 committed by GitHub
commit 473c8abc82
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
33 changed files with 254 additions and 90 deletions

View file

@ -40,6 +40,7 @@ import io.element.android.libraries.textcomposer.model.VoiceMessageState
import io.element.android.libraries.voicerecorder.api.VoiceRecorder
import io.element.android.libraries.voicerecorder.api.VoiceRecorderState
import io.element.android.services.analytics.api.AnalyticsService
import kotlinx.collections.immutable.toPersistentList
import kotlinx.collections.immutable.ImmutableList
import kotlinx.collections.immutable.toImmutableList
import kotlinx.coroutines.CoroutineScope
@ -175,7 +176,7 @@ class VoiceMessageComposerPresenter @Inject constructor(
voiceMessageState = when (val state = recorderState) {
is VoiceRecorderState.Recording -> VoiceMessageState.Recording(
duration = state.elapsedTime,
level = state.level
levels = state.levels.toPersistentList()
)
is VoiceRecorderState.Finished -> VoiceMessageState.Preview(
isSending = isSending,

View file

@ -18,12 +18,13 @@ package io.element.android.features.messages.impl.voicemessages.composer
import androidx.compose.ui.tooling.preview.PreviewParameterProvider
import io.element.android.libraries.textcomposer.model.VoiceMessageState
import kotlinx.collections.immutable.toPersistentList
import kotlin.time.Duration.Companion.seconds
internal open class VoiceMessageComposerStateProvider : PreviewParameterProvider<VoiceMessageComposerState> {
override val values: Sequence<VoiceMessageComposerState>
get() = sequenceOf(
aVoiceMessageComposerState(voiceMessageState = VoiceMessageState.Recording(duration = 61.seconds, level = 0.5f)),
aVoiceMessageComposerState(voiceMessageState = VoiceMessageState.Recording(duration = 61.seconds, levels = aWaveformLevels)),
)
}
@ -35,3 +36,7 @@ internal fun aVoiceMessageComposerState(
showPermissionRationaleDialog = showPermissionRationaleDialog,
eventSink = {},
)
internal var aWaveformLevels = List(100) { it.toFloat() / 100 }.toPersistentList()

View file

@ -44,6 +44,7 @@ import io.element.android.libraries.textcomposer.model.VoiceMessageState
import io.element.android.libraries.voicerecorder.test.FakeVoiceRecorder
import io.element.android.services.analytics.test.FakeAnalyticsService
import io.element.android.tests.testutils.WarmUpRule
import kotlinx.collections.immutable.toPersistentList
import kotlinx.collections.immutable.toImmutableList
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.test.TestScope
@ -67,7 +68,7 @@ class VoiceMessageComposerPresenterTest {
companion object {
private val RECORDING_DURATION = 1.seconds
private val RECORDING_STATE = VoiceMessageState.Recording(RECORDING_DURATION, 0.2f)
private val RECORDING_STATE = VoiceMessageState.Recording(RECORDING_DURATION, listOf(0.1f, 0.2f).toPersistentList())
}
@Test
@ -146,7 +147,7 @@ class VoiceMessageComposerPresenterTest {
}
// Nothing should happen
assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Recording(RECORDING_DURATION, 0.2f))
assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Recording(RECORDING_DURATION, RECORDING_STATE.levels))
voiceRecorder.assertCalls(started = 1, stopped = 0, deleted = 0)
testPauseAndDestroy(finalState)

View file

@ -0,0 +1,57 @@
/*
* Copyright (c) 2023 New Vector Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.element.android.libraries.designsystem.components.media
import androidx.compose.ui.geometry.CornerRadius
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.geometry.Size
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.drawscope.DrawScope
import androidx.compose.ui.graphics.drawscope.Fill
import androidx.compose.ui.unit.Dp
import androidx.compose.ui.unit.DpSize
import androidx.compose.ui.unit.dp
import kotlinx.collections.immutable.ImmutableList
import kotlin.math.max
fun DrawScope.drawWaveform(
waveformData: ImmutableList<Float>,
canvasSize: DpSize,
brush: Brush,
minimumGraphAmplitude: Float = 2F,
lineWidth: Dp = 2.dp,
linePadding: Dp = 2.dp,
) {
val centerY = canvasSize.height.toPx() / 2
val cornerRadius = lineWidth / 2
waveformData.forEachIndexed { index, amplitude ->
val drawingAmplitude = max(minimumGraphAmplitude, amplitude * (canvasSize.height.toPx() - 2))
drawRoundRect(
brush = brush,
topLeft = Offset(
x = index * (linePadding + lineWidth).toPx(),
y = centerY - drawingAmplitude / 2
),
size = Size(
width = lineWidth.toPx(),
height = drawingAmplitude
),
cornerRadius = CornerRadius(cornerRadius.toPx(), cornerRadius.toPx()),
style = Fill
)
}
}

View file

@ -49,7 +49,6 @@ import io.element.android.libraries.theme.ElementTheme
import kotlinx.collections.immutable.ImmutableList
import kotlinx.collections.immutable.persistentListOf
import kotlinx.collections.immutable.toPersistentList
import kotlin.math.max
import kotlin.math.roundToInt
private const val DEFAULT_GRAPHICS_LAYER_ALPHA: Float = 0.99F
@ -67,7 +66,6 @@ private const val DEFAULT_GRAPHICS_LAYER_ALPHA: Float = 0.99F
* @param cursorBrush The brush to use to draw the cursor.
* @param lineWidth The width of the waveform lines.
* @param linePadding The padding between waveform lines.
* @param minimumGraphAmplitude The minimum amplitude to display, regardless of waveform data.
*/
@OptIn(ExperimentalComposeUiApi::class)
@Composable
@ -82,7 +80,6 @@ fun WaveformPlaybackView(
cursorBrush: Brush = SolidColor(ElementTheme.colors.iconAccentTertiary),
lineWidth: Dp = 2.dp,
linePadding: Dp = 2.dp,
minimumGraphAmplitude: Float = 2F,
) {
val seekProgress = remember { mutableStateOf<Float?>(null) }
var canvasSize by remember { mutableStateOf(DpSize(0.dp, 0.dp)) }
@ -139,22 +136,13 @@ fun WaveformPlaybackView(
canvasSizePx = size
val centerY = canvasSize.height.toPx() / 2
val cornerRadius = lineWidth / 2
normalizedWaveformData.forEachIndexed { index, amplitude ->
val drawingAmplitude = max(minimumGraphAmplitude, amplitude * (canvasSize.height.toPx() - 2))
drawRoundRect(
brush = brush,
topLeft = Offset(
x = index * (linePadding + lineWidth).toPx(),
y = centerY - drawingAmplitude / 2
),
size = Size(
width = lineWidth.toPx(),
height = drawingAmplitude
),
cornerRadius = CornerRadius(cornerRadius.toPx(), cornerRadius.toPx()),
style = Fill
)
}
drawWaveform(
waveformData = normalizedWaveformData,
canvasSize = canvasSize,
brush = brush,
lineWidth = lineWidth,
linePadding = linePadding
)
drawRect(
brush = progressBrush,
size = Size(

View file

@ -85,6 +85,7 @@ import io.element.android.wysiwyg.compose.RichTextEditor
import io.element.android.wysiwyg.compose.RichTextEditorState
import kotlinx.collections.immutable.ImmutableList
import kotlinx.collections.immutable.persistentListOf
import kotlinx.collections.immutable.toPersistentList
import kotlin.time.Duration.Companion.seconds
import uniffi.wysiwyg_composer.MenuAction
@ -214,7 +215,7 @@ fun TextComposer(
onSeek = onSeekVoiceMessage,
)
is VoiceMessageState.Recording ->
VoiceMessageRecording(voiceMessageState.level, voiceMessageState.duration)
VoiceMessageRecording(voiceMessageState.levels, voiceMessageState.duration)
VoiceMessageState.Idle -> {}
}
}
@ -814,7 +815,7 @@ internal fun TextComposerVoicePreview() = ElementPreview {
enableVoiceMessages = true,
)
PreviewColumn(items = persistentListOf({
VoicePreview(voiceMessageState = VoiceMessageState.Recording(61.seconds, 0.5f))
VoicePreview(voiceMessageState = VoiceMessageState.Recording(61.seconds, List(100) { it.toFloat() / 100 }.toPersistentList()))
}, {
VoicePreview(voiceMessageState = VoiceMessageState.Preview(isSending = false, isPlaying = false, waveform = createFakeWaveform()))
}, {

View file

@ -0,0 +1,109 @@
/*
* Copyright (c) 2023 New Vector Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.element.android.libraries.textcomposer.components
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.width
import androidx.compose.runtime.Composable
import androidx.compose.runtime.derivedStateOf
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableIntStateOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.SolidColor
import androidx.compose.ui.graphics.graphicsLayer
import androidx.compose.ui.layout.onSizeChanged
import androidx.compose.ui.unit.Dp
import androidx.compose.ui.unit.DpSize
import androidx.compose.ui.unit.dp
import io.element.android.libraries.designsystem.components.media.drawWaveform
import io.element.android.libraries.designsystem.preview.ElementPreview
import io.element.android.libraries.designsystem.preview.PreviewsDayNight
import io.element.android.libraries.theme.ElementTheme
import kotlinx.collections.immutable.ImmutableList
import kotlinx.collections.immutable.toPersistentList
import java.lang.Float.min
private const val DEFAULT_GRAPHICS_LAYER_ALPHA: Float = 0.99F
private val waveFormHeight = 26.dp
@Composable
fun LiveWaveformView(
levels: ImmutableList<Float>,
modifier: Modifier = Modifier,
brush: Brush = SolidColor(ElementTheme.colors.iconQuaternary),
lineWidth: Dp = 2.dp,
linePadding: Dp = 2.dp,
) {
var canvasSize by remember { mutableStateOf(DpSize(0.dp, 0.dp)) }
var parentWidth by remember { mutableIntStateOf(0) }
val waveformWidth by remember(levels, lineWidth, linePadding) {
derivedStateOf {
levels.size * (lineWidth.value + linePadding.value)
}
}
Box(contentAlignment = Alignment.CenterEnd,
modifier = modifier
.fillMaxWidth()
.height(waveFormHeight)
.onSizeChanged { parentWidth = it.width }
) {
Canvas(
modifier = Modifier
.width(Dp(waveformWidth))
.graphicsLayer(alpha = DEFAULT_GRAPHICS_LAYER_ALPHA)
.then(modifier)
) {
canvasSize = DpSize(Dp(min(waveformWidth, parentWidth.toFloat())), size.height.toDp())
val countThatFitsWidth = (parentWidth.toFloat() / (lineWidth.toPx() + linePadding.toPx())).toInt()
drawWaveform(
waveformData = levels.takeLast(countThatFitsWidth).toPersistentList(),
canvasSize = canvasSize,
brush = brush,
lineWidth = lineWidth,
linePadding = linePadding,
)
}
}
}
@PreviewsDayNight
@Composable
internal fun LiveWaveformViewPreview() = ElementPreview {
Column {
LiveWaveformView(
levels = List(100) { it.toFloat() / 100 }.toPersistentList(),
modifier = Modifier.height(34.dp),
)
LiveWaveformView(
levels = List(40) { it.toFloat() / 40 }.toPersistentList(),
modifier = Modifier.height(34.dp),
)
}
}

View file

@ -20,7 +20,6 @@ import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxHeight
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.heightIn
@ -37,12 +36,14 @@ import io.element.android.libraries.designsystem.preview.PreviewsDayNight
import io.element.android.libraries.designsystem.theme.components.Text
import io.element.android.libraries.theme.ElementTheme
import io.element.android.libraries.ui.utils.time.formatShort
import kotlinx.collections.immutable.ImmutableList
import kotlinx.collections.immutable.toPersistentList
import kotlin.time.Duration
import kotlin.time.Duration.Companion.seconds
@Composable
internal fun VoiceMessageRecording(
level: Float,
levels: ImmutableList<Float>,
duration: Duration,
modifier: Modifier = Modifier,
) {
@ -70,28 +71,11 @@ internal fun VoiceMessageRecording(
Spacer(Modifier.size(20.dp))
// TODO Replace with waveform UI
DebugAudioLevel(
modifier = Modifier.weight(1f), level = level
)
}
}
@Composable
private fun DebugAudioLevel(
level: Float,
modifier: Modifier = Modifier,
) {
Box(
modifier = modifier
.height(26.dp)
) {
Box(
LiveWaveformView(
modifier = Modifier
.align(Alignment.CenterEnd)
.fillMaxWidth(level)
.background(ElementTheme.colors.iconQuaternary, shape = MaterialTheme.shapes.small)
.fillMaxHeight()
.height(26.dp)
.weight(1f),
levels = levels
)
}
}
@ -108,5 +92,5 @@ private fun RedRecordingDot(
@PreviewsDayNight
@Composable
internal fun VoiceMessageRecordingPreview() = ElementPreview {
VoiceMessageRecording(0.5f, 0.seconds)
VoiceMessageRecording(List(100) { it.toFloat() / 100 }.toPersistentList(), 0.seconds)
}

View file

@ -30,6 +30,6 @@ sealed class VoiceMessageState {
data class Recording(
val duration: Duration,
val level: Float,
val levels: ImmutableList<Float>,
): VoiceMessageState()
}

View file

@ -29,9 +29,9 @@ sealed class VoiceRecorderState {
* The recorder is currently recording.
*
* @property elapsedTime The elapsed time since the recording started.
* @property level The current audio level of the recording as a fraction of 1.
* @property levels The current audio levels of the recording as a fraction of 1.
*/
data class Recording(val elapsedTime: Duration, val level: Float) : VoiceRecorderState()
data class Recording(val elapsedTime: Duration, val levels: List<Float>) : VoiceRecorderState()
/**
* The recorder has finished recording.

View file

@ -38,6 +38,8 @@ import kotlinx.coroutines.Job
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.yield
import timber.log.Timber
import java.io.File
@ -67,6 +69,7 @@ class VoiceRecorderImpl @Inject constructor(
private var audioReader: AudioReader? = null
private var recordingJob: Job? = null
private val levels: MutableList<Float> = mutableListOf()
private val lock = Mutex()
private val _state = MutableStateFlow<VoiceRecorderState>(VoiceRecorderState.Idle)
override val state: StateFlow<VoiceRecorderState> = _state
@ -76,7 +79,10 @@ class VoiceRecorderImpl @Inject constructor(
Timber.i("Voice recorder started recording")
outputFile = fileManager.createFile()
.also(encoder::init)
levels.clear()
lock.withLock {
levels.clear()
}
val audioRecorder = audioReaderFactory.create(config, dispatchers).also { audioReader = it }
@ -96,13 +102,16 @@ class VoiceRecorderImpl @Inject constructor(
when (audio) {
is Audio.Data -> {
val audioLevel = audioLevelCalculator.calculateAudioLevel(audio.buffer)
_state.emit(VoiceRecorderState.Recording(elapsedTime, audioLevel))
levels.add(audioLevel)
lock.withLock{
levels.add(audioLevel)
_state.emit(VoiceRecorderState.Recording(elapsedTime, levels.toList()))
}
encoder.encode(audio.buffer, audio.readSize)
}
is Audio.Error -> {
Timber.e("Voice message error: code=${audio.audioRecordErrorCode}")
_state.emit(VoiceRecorderState.Recording(elapsedTime, 0.0f))
_state.emit(VoiceRecorderState.Recording(elapsedTime, listOf()))
}
}
}
@ -126,21 +135,24 @@ class VoiceRecorderImpl @Inject constructor(
audioReader = null
encoder.release()
if (cancelled) {
deleteRecording()
levels.clear()
}
_state.emit(
when (val file = outputFile) {
null -> VoiceRecorderState.Idle
else -> VoiceRecorderState.Finished(
file = file,
mimeType = fileConfig.mimeType,
waveform = levels.resample(100),
)
lock.withLock {
if (cancelled) {
deleteRecording()
levels.clear()
}
)
_state.emit(
when (val file = outputFile) {
null -> VoiceRecorderState.Idle
else -> VoiceRecorderState.Finished(
file = file,
mimeType = fileConfig.mimeType,
waveform = levels.resample(100),
)
}
)
}
}
/**

View file

@ -60,11 +60,11 @@ class VoiceRecorderImplTest {
assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Idle)
voiceRecorder.startRecord()
assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(0.seconds, 1.0f))
assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(0.seconds, listOf(1.0f)))
timeSource += 1.seconds
assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(1.seconds, 0.0f))
assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(1.seconds, listOf()))
timeSource += 1.seconds
assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(2.seconds, 1.0f))
assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(2.seconds, listOf(1.0f, 1.0f)))
}
}
@ -75,9 +75,9 @@ class VoiceRecorderImplTest {
assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Idle)
voiceRecorder.startRecord()
assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(0.minutes, 1.0f))
assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(0.minutes, listOf(1.0f)))
timeSource += 29.minutes
assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(29.minutes, 0.0f))
assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(29.minutes, listOf()))
timeSource += 1.minutes
assertThat(awaitItem()).isEqualTo(

View file

@ -54,8 +54,8 @@ class FakeVoiceRecorder(
curRecording = File("file.ogg")
timeSource += recordingDuration
levels.forEach {
_state.emit(VoiceRecorderState.Recording(startedAt.elapsedNow(), it))
for(i in 1..levels.size) {
_state.emit(VoiceRecorderState.Recording(startedAt.elapsedNow(), levels.take(i)))
}
}

View file

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3c25252b8d43f4ffb58673f375709b4811901a78676580a7dd0288b8624615d7
size 7787
oid sha256:3d5209087d7841a80f7213f26aaf8322cc5fee03466b9059e06b0daf5f489c1b
size 10012

View file

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:55fa9c5633d3776a3401db72e2e53d9eacedd745e0db7f259f6ada5d7a8d584a
size 7473
oid sha256:5b8fca1de81e424f01a518e3a2547b58c0e9eb7249327f3fcdb1cf62ab655972
size 9429

View file

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c8ab81200d74af2e45390223484cf17c6bb488eedf3cb10a425e3e2b7546e1b0
size 11422

View file

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:47fc7fcebbe8c3f8ad745eac3e9687c7722a898e55e1f11f2903cffa2af00636
size 11059

View file

@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:208ad62e23efd6f07e2cce08c1d1511af4c4fc2ae6bc3299134fed9efb1c55d3
size 7238

View file

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8ee23864465d4f6cd09f4d9c187916793ce4c8fce2ad77a29e2321e1caa57b7d
size 10135

View file

@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8cf7662c33de6ad1b58785674cbfce1d6323fe84e35c8398b563ea65e5ff9fa7
size 6919

View file

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:408ee43f0955bcb120f7e3fbbc51b6276ae139863e1691e6d768a35005d3fa1b
size 9498

View file

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8fbbbac71323947df0f09b8614b23206add7dbdd36a629668f3474be8b89b5a8
size 25549
oid sha256:5abbadadbc4b0943be47d269cddecfe42e8a7ae43e4dd618eb2a8d5aefc7c1b1
size 27291

View file

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:81863e643154025c515efb0070ceaad2e04b74517ddbd314d214d7d7dae2286d
size 23193
oid sha256:f73f8eed6746a54bd0bf706acda638155ea99e7aba8b78fc809b9f26b0a5986f
size 24901