This repository has no description
perf: replace software video encoding with CameraX VideoCapture
Use CameraX VideoCapture use case for hardware-accelerated recording
instead of per-frame ARGB→NV12 software conversion via MediaCodec.
This eliminates the CPU-intensive bitmap conversion and encoding that
was competing with pose/object detection on every frame.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
author
virtualintern
co-author
Claude Opus 4.6 (1M context)
committer
![]()
tangled.org
date
(Apr 1, 2026, 8:15 PM UTC)
commit
cfab42ed
cfab42ed1b747606f668d7a0bd1beee155dd4548
parent
3efaf6ae
3efaf6ae8b04a01065b88fe08773a27a84297ceb
+79
-104
+79
-104
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
+79
-104
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
···
12
12
import androidx.camera.core.ImageAnalysis
13
13
import androidx.camera.core.Preview
14
14
import androidx.camera.lifecycle.ProcessCameraProvider
15
+
import androidx.camera.video.FileOutputOptions
16
+
import androidx.camera.video.Quality
17
+
import androidx.camera.video.QualitySelector
18
+
import androidx.camera.video.Recorder
19
+
import androidx.camera.video.Recording
20
+
import androidx.camera.video.VideoCapture
21
+
import androidx.camera.video.VideoRecordEvent
15
22
import androidx.camera.view.PreviewView
16
23
import androidx.compose.foundation.Canvas
17
24
import androidx.compose.foundation.layout.Box
···
21
28
import androidx.compose.runtime.getValue
22
29
import androidx.compose.runtime.mutableStateOf
23
30
import androidx.compose.runtime.remember
24
-
import androidx.compose.runtime.rememberCoroutineScope
25
31
import androidx.compose.runtime.setValue
26
32
import androidx.compose.ui.Modifier
27
33
import androidx.compose.ui.draw.clipToBounds
···
34
40
import androidx.compose.ui.graphics.asAndroidBitmap
35
41
import androidx.compose.ui.graphics.asImageBitmap
36
42
import androidx.compose.ui.graphics.drawscope.Stroke
43
+
import androidx.core.content.ContextCompat
37
44
import androidx.compose.ui.platform.LocalContext
38
45
import androidx.compose.ui.text.rememberTextMeasurer
39
46
import androidx.compose.ui.viewinterop.AndroidView
···
49
56
import com.performancecoachlab.posedetection.recording.AnalysisObject
50
57
import com.performancecoachlab.posedetection.skeleton.Skeleton
51
58
import com.performancecoachlab.posedetection.skeleton.SkeletonRepository
52
-
import kotlinx.coroutines.launch
53
59
import java.io.File
54
60
import java.util.concurrent.Executors
55
61
import java.util.concurrent.atomic.AtomicBoolean
56
62
import java.util.concurrent.atomic.AtomicLong
57
-
58
-
// Data class to hold recording state for each recording ID
59
-
data class RecordingSlot(
60
-
var builder: com.performancecoachlab.posedetection.encoding.VideoBuilder? = null,
61
-
var firstTimestampMs: Long? = null,
62
-
val outputPath: String
63
-
)
64
63
65
64
@OptIn(ExperimentalCamera2Interop::class)
66
65
private fun buildBackUltraWideSelectorOrNull(cameraProvider: ProcessCameraProvider): CameraSelector? {
···
139
138
val lifecycleOwner: LifecycleOwner = LocalLifecycleOwner.current
140
139
val previewView: PreviewView = remember { PreviewView(context) }
141
140
val executor = remember { Executors.newSingleThreadExecutor() }
142
-
val scope = rememberCoroutineScope()
143
141
144
142
var focus by remember { mutableStateOf(focusArea) }
145
143
var objectDetector by remember { mutableStateOf(objectModel) }
···
158
156
objectDetector = objectModel
159
157
}
160
158
161
-
// Multi-recording state - map of recording ID to recording slot
162
-
var activeRecordings by remember { mutableStateOf<Map<String, RecordingSlot>>(emptyMap()) }
159
+
// CameraX VideoCapture recording state
160
+
var recorderRef by remember { mutableStateOf<Recorder?>(null) }
161
+
var currentCxRecording by remember { mutableStateOf<Recording?>(null) }
162
+
var currentRecordingId by remember { mutableStateOf<String?>(null) }
163
+
var currentRecordingPath by remember { mutableStateOf<String?>(null) }
163
164
164
-
// Restored helpers (were accidentally removed during refactor)
165
-
suspend fun startRecording(id: String, width: Int, height: Int) {
166
-
val outputPath = File(
167
-
context.cacheDir,
168
-
"camera_recording_${id}_${System.currentTimeMillis()}.mp4"
169
-
).absolutePath
170
-
val slot = RecordingSlot(
171
-
builder = null, // Lazily created on first frame
172
-
firstTimestampMs = null,
173
-
outputPath = outputPath
174
-
)
175
-
activeRecordings = activeRecordings + (id to slot)
176
-
}
165
+
// React to recordingId changes — start/stop CameraX VideoCapture recording
166
+
LaunchedEffect(recordingId, recorderRef) {
167
+
val recorder = recorderRef ?: return@LaunchedEffect
177
168
178
-
suspend fun stopRecording(id: String) {
179
-
activeRecordings[id]?.let { slot ->
180
-
slot.builder?.let { builder ->
181
-
try {
182
-
val path = builder.finalize()
183
-
onVideoSaved(id, path)
184
-
} catch (_: Exception) {
185
-
// ignore finalize errors
186
-
}
187
-
}
188
-
activeRecordings = activeRecordings - id
169
+
// Stop any active recording when recordingId changes.
170
+
currentCxRecording?.let { activeRec ->
171
+
activeRec.stop()
172
+
currentCxRecording = null
189
173
}
190
-
}
191
174
192
-
// Recording no longer depends on a continuously updated UI bitmap.
193
-
fun addFrameToActiveRecordings(frame: ImageBitmap, timestampMs: Long) {
194
-
if (activeRecordings.isNotEmpty()) {
195
-
scope.launch {
196
-
activeRecordings.forEach { (id, slot) ->
197
-
try {
198
-
// Lazily create builder on first frame to lock width/height
199
-
if (slot.builder == null) {
200
-
slot.builder =
201
-
com.performancecoachlab.posedetection.encoding.createVideoBuilder(
202
-
outputPath = slot.outputPath,
203
-
fps = 30,
204
-
width = frame.width,
205
-
height = frame.height
206
-
)
207
-
slot.firstTimestampMs = timestampMs
208
-
}
175
+
// Start new recording if recordingId is non-null.
176
+
recordingId?.let { id ->
177
+
val outputPath = File(
178
+
context.cacheDir,
179
+
"camera_recording_${id}_${System.currentTimeMillis()}.mp4"
180
+
).absolutePath
181
+
val outputFile = File(outputPath)
182
+
val fileOutputOptions = FileOutputOptions.Builder(outputFile).build()
183
+
184
+
currentRecordingId = id
185
+
currentRecordingPath = outputPath
186
+
187
+
val pendingRecording = recorder.prepareRecording(context, fileOutputOptions)
209
188
210
-
slot.builder?.let { builder ->
211
-
val relativeTimestamp =
212
-
timestampMs - (slot.firstTimestampMs ?: timestampMs)
213
-
builder.addFrame(frame, relativeTimestamp)
189
+
currentCxRecording = pendingRecording.start(
190
+
ContextCompat.getMainExecutor(context)
191
+
) { event ->
192
+
when (event) {
193
+
is VideoRecordEvent.Start -> {
194
+
onRecordToggled(true)
195
+
}
196
+
is VideoRecordEvent.Finalize -> {
197
+
onRecordToggled(false)
198
+
currentRecordingId?.let { savedId ->
199
+
currentRecordingPath?.let { savedPath ->
200
+
onVideoSaved(savedId, savedPath)
201
+
}
214
202
}
215
-
} catch (e: Exception) {
216
-
// Handle recording errors gracefully
217
-
//println("Error recording frame for ID $id: ${e.message}")
203
+
currentCxRecording = null
204
+
currentRecordingId = null
205
+
currentRecordingPath = null
218
206
}
219
207
}
220
208
}
221
209
}
222
210
}
223
211
224
-
// React to recordingIds changes - diff and start/stop recordings
225
-
LaunchedEffect(recordingId) {
226
-
val currentIds = activeRecordings.keys
227
-
recordingId?.also { id ->
228
-
currentIds.forEach {
229
-
if (!it.contentEquals(id)) {
230
-
stopRecording(it)
231
-
}
232
-
}
233
-
if (!currentIds.contains(id)) {
234
-
val size = analysisSize
235
-
if (size != null) {
236
-
startRecording(id, size.width.toInt(), size.height.toInt())
237
-
} else {
238
-
// If no frame yet, create empty slot that will be initialized on first frame
239
-
val outputPath = File(
240
-
context.cacheDir,
241
-
"camera_recording_${id}_${System.currentTimeMillis()}.mp4"
242
-
).absolutePath
243
-
val slot = RecordingSlot(
244
-
builder = null,
245
-
firstTimestampMs = null,
246
-
outputPath = outputPath
247
-
)
248
-
activeRecordings = activeRecordings + (id to slot)
249
-
}
250
-
}
251
-
} ?: run {
252
-
currentIds.forEach { stopRecording(it) }
253
-
}
254
-
}
255
-
256
212
val gate = remember { FrameGate() }
257
213
258
214
LaunchedEffect(lifecycleOwner, frontCamera, useUltraWide, previewFillMode) {
215
+
// Stop any in-progress recording before rebinding camera.
216
+
currentCxRecording?.stop()
217
+
currentCxRecording = null
218
+
259
219
val cameraProvider = ProcessCameraProvider.getInstance(context).get()
260
220
cameraProvider.unbindAll()
261
221
···
344
304
)
345
305
}
346
306
347
-
// Only convert to ImageBitmap when we're actually recording.
348
-
if (activeRecordings.isNotEmpty()) {
349
-
val inbmp = frameBitmap.asImageBitmap()
350
-
addFrameToActiveRecordings(inbmp, now)
351
-
}
352
307
} finally {
353
308
gate.exit()
354
309
}
355
310
}
356
311
}
357
312
}
358
-
val camera = cameraProvider.bindToLifecycle(
359
-
lifecycleOwner,
360
-
cameraSelector,
361
-
preview,
362
-
imageAnalysis
363
-
)
313
+
val recorder = Recorder.Builder()
314
+
.setQualitySelector(QualitySelector.from(Quality.HIGHEST))
315
+
.build()
316
+
val videoCapture = VideoCapture.withOutput(recorder)
317
+
318
+
val camera = try {
319
+
cameraProvider.bindToLifecycle(
320
+
lifecycleOwner,
321
+
cameraSelector,
322
+
preview,
323
+
imageAnalysis,
324
+
videoCapture
325
+
).also {
326
+
recorderRef = recorder
327
+
}
328
+
} catch (e: IllegalArgumentException) {
329
+
// Fallback for devices that cannot bind 3 use cases simultaneously.
330
+
Logger.w(e) { "Cannot bind VideoCapture; recording disabled on this device." }
331
+
recorderRef = null
332
+
cameraProvider.bindToLifecycle(
333
+
lifecycleOwner,
334
+
cameraSelector,
335
+
preview,
336
+
imageAnalysis
337
+
)
338
+
}
364
339
365
340
// Helpful debug: see which physical camera ID CameraX picked.
366
341
runCatching {