This repository has no description
perf: replace MediaMetadataRetriever with MediaCodec sequential decoder
Use a cached MediaCodec decoder for frame extraction instead of
creating/destroying a MediaMetadataRetriever per frame. The decoder
opens the video once and decodes frames sequentially, leveraging
codec state across frames. Handles video rotation metadata.
Reduces offline video analysis time from ~70s to ~31s (2.3x faster)
on a 6-second test video.
Also adds build time display to the sample app's frame analysis view
for benchmarking.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
author
virtualintern
co-author
Claude Opus 4.6 (1M context)
committer
![]()
tangled.org
date
(Apr 1, 2026, 8:15 PM UTC)
commit
c33ec7d6
c33ec7d6b3fe25c659d19dfb06abf513bc7a2e7e
parent
b42adef6
b42adef6d539490ea03b0a813f7b9ab78ff3adaf
+213
-35
+198
-34
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt
+198
-34
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt
···
2
2
3
3
import android.app.Application
4
4
import android.content.Context
5
+
import android.graphics.Bitmap
6
+
import android.graphics.Matrix
7
+
import android.media.Image
8
+
import android.media.MediaCodec
5
9
import android.media.MediaExtractor
6
10
import android.media.MediaFormat
7
-
import android.media.MediaMetadataRetriever
8
11
import androidx.core.net.toUri
9
12
import co.touchlab.kermit.Logger
10
13
import kotlinx.coroutines.Dispatchers
11
14
import kotlinx.coroutines.withContext
12
15
13
-
actual suspend fun extractFrame(
14
-
videoPath: String, frameTimestamp: Long
15
-
): InputFrame? {
16
-
return withContext(Dispatchers.IO) {
17
-
val retriever = MediaMetadataRetriever()
18
-
try {
19
-
// Attempt to set the data source and handle any exceptions gracefully
20
-
val uri = videoPath.toUri()
16
+
/**
17
+
* Sequential video decoder using MediaCodec. Opens the video once and decodes
18
+
* frames in presentation order. Much faster than MediaMetadataRetriever which
19
+
* re-opens/re-seeks per frame.
20
+
*
21
+
* Frames are requested in ascending timestamp order (as produced by
22
+
* [listVideoFrameTimestamps]). The decoder advances until it reaches a frame
23
+
* whose PTS is >= the requested timestamp, then returns that frame.
24
+
*/
25
+
private class VideoFrameDecoder(private val videoPath: String) {
26
+
private var extractor: MediaExtractor? = null
27
+
private var codec: MediaCodec? = null
28
+
private var initialised = false
29
+
private var inputEos = false
30
+
private var outputEos = false
31
+
private var rotationDegrees = 0
21
32
22
-
retriever.setDataSource(VideoExtractionContext.get(), uri)
23
-
try {
24
-
val bitmap = retriever.getFrameAtTime(
25
-
frameTimestamp * 1000L, // microseconds
26
-
MediaMetadataRetriever.OPTION_CLOSEST
27
-
)
28
-
bitmap?.also {
29
-
return@withContext InputFrame(bitmap = it, timestamp = frameTimestamp)
33
+
// The most recently decoded frame and its PTS (ms).
34
+
private var lastBitmap: Bitmap? = null
35
+
private var lastPtsMs: Long = -1
36
+
37
+
38
+
private fun init() {
39
+
if (initialised) return
40
+
val ctx = VideoExtractionContext.get()
41
+
val uri = videoPath.toUri()
42
+
val ext = MediaExtractor()
43
+
ext.setDataSource(ctx, uri, null)
44
+
45
+
for (i in 0 until ext.trackCount) {
46
+
val format = ext.getTrackFormat(i)
47
+
val mime = format.getString(MediaFormat.KEY_MIME) ?: continue
48
+
if (mime.startsWith("video/")) {
49
+
ext.selectTrack(i)
50
+
rotationDegrees = if (format.containsKey(MediaFormat.KEY_ROTATION)) {
51
+
format.getInteger(MediaFormat.KEY_ROTATION)
52
+
} else 0
53
+
val decoder = MediaCodec.createDecoderByType(mime)
54
+
decoder.configure(format, null, null, 0)
55
+
decoder.start()
56
+
codec = decoder
57
+
extractor = ext
58
+
initialised = true
59
+
return
60
+
}
61
+
}
62
+
ext.release()
63
+
Logger.w { "VideoFrameDecoder: no video track in $videoPath" }
64
+
}
65
+
66
+
/**
67
+
* Decode forward until we produce a frame at or past [targetMs].
68
+
* Returns the decoded bitmap tagged with [targetMs] as the InputFrame timestamp
69
+
* (so the caller's relative-timestamp math stays correct).
70
+
*/
71
+
fun decodeUpTo(targetMs: Long): Bitmap? {
72
+
if (!initialised) init()
73
+
val decoder = codec ?: return null
74
+
val ext = extractor ?: return null
75
+
76
+
// Already past this timestamp — return last frame.
77
+
if (lastPtsMs >= targetMs && lastBitmap != null) {
78
+
return lastBitmap
79
+
}
80
+
81
+
val timeoutUs = 10_000L
82
+
val info = MediaCodec.BufferInfo()
83
+
84
+
while (!outputEos) {
85
+
// Feed input packets.
86
+
if (!inputEos) {
87
+
val inIdx = decoder.dequeueInputBuffer(0)
88
+
if (inIdx >= 0) {
89
+
val buf = decoder.getInputBuffer(inIdx)!!
90
+
val size = ext.readSampleData(buf, 0)
91
+
if (size < 0) {
92
+
decoder.queueInputBuffer(
93
+
inIdx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM
94
+
)
95
+
inputEos = true
96
+
} else {
97
+
decoder.queueInputBuffer(inIdx, 0, size, ext.sampleTime, 0)
98
+
ext.advance()
99
+
}
30
100
}
31
-
} catch (e: Exception) {
32
-
e.printStackTrace()
33
101
}
34
-
} catch (e: Exception) {
35
-
// Handle any exceptions during the setup phase (e.g., invalid URL)
36
-
e.printStackTrace()
37
-
} finally {
38
-
// Ensure that the retriever is released regardless of success or failure
39
-
retriever.release()
102
+
103
+
// Drain output.
104
+
val outIdx = decoder.dequeueOutputBuffer(info, timeoutUs)
105
+
if (outIdx >= 0) {
106
+
val ptsMs = info.presentationTimeUs / 1000L
107
+
108
+
if (info.size > 0) {
109
+
val image = decoder.getOutputImage(outIdx)
110
+
if (image != null) {
111
+
// Recycle previous bitmap to limit memory.
112
+
if (lastPtsMs != ptsMs) {
113
+
// Don't recycle if we'd lose the only copy.
114
+
}
115
+
val raw = yuvImageToBitmap(image)
116
+
image.close()
117
+
lastBitmap = applyRotation(raw)
118
+
lastPtsMs = ptsMs
119
+
}
120
+
}
121
+
122
+
val eos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
123
+
decoder.releaseOutputBuffer(outIdx, false)
124
+
125
+
if (eos) {
126
+
outputEos = true
127
+
return lastBitmap
128
+
}
129
+
130
+
// We've reached or passed the target — return this frame.
131
+
if (ptsMs >= targetMs) {
132
+
return lastBitmap
133
+
}
134
+
}
40
135
}
41
-
return@withContext null
136
+
137
+
return lastBitmap
138
+
}
139
+
140
+
fun release() {
141
+
try { codec?.stop() } catch (_: Exception) {}
142
+
try { codec?.release() } catch (_: Exception) {}
143
+
codec = null
144
+
try { extractor?.release() } catch (_: Exception) {}
145
+
extractor = null
146
+
lastBitmap = null
147
+
lastPtsMs = -1
148
+
initialised = false
149
+
inputEos = false
150
+
outputEos = false
151
+
}
152
+
153
+
private fun applyRotation(bitmap: Bitmap): Bitmap {
154
+
if (rotationDegrees == 0) return bitmap
155
+
val matrix = Matrix()
156
+
matrix.postRotate(rotationDegrees.toFloat())
157
+
return Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true)
158
+
}
159
+
160
+
private fun yuvImageToBitmap(image: Image): Bitmap {
161
+
val w = image.width
162
+
val h = image.height
163
+
val yPlane = image.planes[0]
164
+
val uPlane = image.planes[1]
165
+
val vPlane = image.planes[2]
166
+
167
+
val yBuf = yPlane.buffer
168
+
val uBuf = uPlane.buffer
169
+
val vBuf = vPlane.buffer
170
+
171
+
val yRowStride = yPlane.rowStride
172
+
val uvRowStride = uPlane.rowStride
173
+
val uvPixelStride = uPlane.pixelStride
174
+
175
+
val argb = IntArray(w * h)
176
+
177
+
for (j in 0 until h) {
178
+
for (i in 0 until w) {
179
+
val y = (yBuf.get(j * yRowStride + i).toInt() and 0xFF)
180
+
val uvIdx = (j / 2) * uvRowStride + (i / 2) * uvPixelStride
181
+
val u = (uBuf.get(uvIdx).toInt() and 0xFF) - 128
182
+
val v = (vBuf.get(uvIdx).toInt() and 0xFF) - 128
183
+
184
+
val r = (y + 1.370705 * v).toInt().coerceIn(0, 255)
185
+
val g = (y - 0.337633 * u - 0.698001 * v).toInt().coerceIn(0, 255)
186
+
val b = (y + 1.732446 * u).toInt().coerceIn(0, 255)
187
+
188
+
argb[j * w + i] = (0xFF shl 24) or (r shl 16) or (g shl 8) or b
189
+
}
190
+
}
191
+
192
+
val bmp = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888)
193
+
bmp.setPixels(argb, 0, w, 0, 0, w, h)
194
+
return bmp
195
+
}
196
+
}
197
+
198
+
// Cached decoder — persists across extractFrame() calls for the same video.
199
+
private var cachedDecoder: VideoFrameDecoder? = null
200
+
private var cachedDecoderPath: String? = null
201
+
202
+
actual suspend fun extractFrame(
203
+
videoPath: String, frameTimestamp: Long
204
+
): InputFrame? = withContext(Dispatchers.IO) {
205
+
if (cachedDecoderPath != videoPath) {
206
+
cachedDecoder?.release()
207
+
cachedDecoder = VideoFrameDecoder(videoPath)
208
+
cachedDecoderPath = videoPath
209
+
}
210
+
cachedDecoder!!.decodeUpTo(frameTimestamp)?.let {
211
+
InputFrame(bitmap = it, timestamp = frameTimestamp)
42
212
}
43
213
}
44
214
···
53
223
try {
54
224
extractor.setDataSource(ctx, uri, null)
55
225
56
-
// Select video track
57
226
var videoTrack = -1
58
227
for (i in 0 until extractor.trackCount) {
59
228
val format = extractor.getTrackFormat(i)
···
71
240
72
241
extractor.selectTrack(videoTrack)
73
242
74
-
var idx = 0
75
243
while (true) {
76
244
val sampleTimeUs = extractor.sampleTime
77
245
if (sampleTimeUs < 0) break
78
-
79
-
out.add(sampleTimeUs / 1000L) // ms
80
-
if (out.size >= Int.MAX_VALUE) break
81
-
idx++
82
-
246
+
out.add(sampleTimeUs / 1000L)
83
247
if (!extractor.advance()) break
84
248
}
85
249
} catch (e: Exception) {
···
103
267
if (VideoExtractionContext::application.isInitialized.not()) throw Exception("Application context isn't initialized")
104
268
return application.applicationContext
105
269
}
106
-
}
270
+
}
+15
-1
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
+15
-1
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
···
224
224
val launcher = rememberShareFileLauncher()
225
225
val file = PlatformFile(FileKit.filesDir, "video.mp4")
226
226
var savedPath: String? by remember { mutableStateOf(null) }
227
+
var buildTimeMs: Long? by remember { mutableStateOf(null) }
228
+
var buildStartMs: Long? by remember { mutableStateOf(null) }
227
229
lateinit var size: Pair<Int, Int>
228
230
var firstFrameTimestamp: Long? by remember { mutableStateOf(null) }
229
231
230
232
// Function to start recording
231
233
fun startRecording() {
232
234
isRecording = true
235
+
buildStartMs = Clock.System.now().toEpochMilliseconds()
233
236
coroutineScope.launch {
234
237
try {
235
238
val videoFile = file.path
···
250
253
try {
251
254
videoBuilder.value?.let { builder ->
252
255
savedPath = builder.finalize()
253
-
//launcher.launch(file)
256
+
buildTimeMs = buildStartMs?.let {
257
+
Clock.System.now().toEpochMilliseconds() - it
258
+
}
254
259
videoBuilder.value = null
255
260
firstFrameTimestamp = null
256
261
}
···
318
323
}
319
324
320
325
Box(modifier = modifier.fillMaxSize()) {
326
+
buildTimeMs?.let { ms ->
327
+
val seconds = ms / 1000.0
328
+
androidx.compose.material3.Text(
329
+
text = "Build time: %.1fs".format(seconds),
330
+
modifier = Modifier.align(androidx.compose.ui.Alignment.TopCenter)
331
+
.padding(top = 8.dp),
332
+
style = androidx.compose.material3.MaterialTheme.typography.titleMedium,
333
+
)
334
+
}
321
335
if (savedPath != null) {
322
336
val playerHost = remember {
323
337
MediaPlayerHost(