feat: split recordings seamlessly · nateholland.bsky.social/PoseDetection@43f3e2e

+86 -36

posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt

··· 55 55 import org.tensorflow.lite.support.image.TensorImage 56 56 import org.tensorflow.lite.task.vision.detector.ObjectDetector 57 57 58 + // Data class to hold recording state for each recording ID 59 + data class RecordingSlot( 60 + var builder: com.performancecoachlab.posedetection.encoding.VideoBuilder? = null, 61 + var firstTimestampMs: Long? = null, 62 + val outputPath: String 63 + ) 64 + 58 65 @OptIn(ExperimentalGetImage::class) 59 66 @Composable 60 67 actual fun CameraView( ··· 66 73 drawObjects: ((List<AnalysisObject>) -> List<DrawableObject>)?, 67 74 modifier: Modifier, 68 75 frontCamera: Boolean, 69 - isRecording: Boolean, 76 + recordingId: String?, 70 77 focusArea: Rect?, 71 78 onRecordToggled: (Boolean) -> Unit, 72 - onVideoSaved: (String) -> Unit, 79 + onVideoSaved: (String, String) -> Unit, 73 80 ) { 74 81 var bitmap by remember { mutableStateOf<ImageBitmap?>(null) } 75 82 val options = ··· 80 87 val previewView: PreviewView = remember { PreviewView(context) } 81 88 val executor = remember { Executors.newSingleThreadExecutor() } 82 89 val scope = rememberCoroutineScope() 83 - var firstFrameTimestamp: Long? = null 84 90 var focus by remember { mutableStateOf(focusArea) } 85 91 var objectDetector by remember { mutableStateOf(objectModel) } 86 92 var currentDetectMode by remember { mutableStateOf(detectMode) } ··· 96 102 objectDetector = objectModel 97 103 } 98 104 99 - // Video recording state 100 - var videoBuilder by remember { mutableStateOf<com.performancecoachlab.posedetection.encoding.VideoBuilder?>(null) } 105 + // Multi-recording state - map of recording ID to recording slot 106 + var activeRecordings by remember { mutableStateOf<Map<String, RecordingSlot>>(emptyMap()) } 101 107 102 - var recordingActive by remember { mutableStateOf(false) } 103 - var videoSavedPath by remember { mutableStateOf<String?>(null) } 104 - 105 - suspend fun startRecording(width: Int, height: Int) { 106 - videoBuilder = com.performancecoachlab.posedetection.encoding.createVideoBuilder( 107 - outputPath = File(context.cacheDir, "camera_recording_${System.currentTimeMillis()}.mp4").absolutePath, 108 - fps = 30, 109 - width = width, 110 - height = height 108 + suspend fun startRecording(id: String, width: Int, height: Int) { 109 + val outputPath = File(context.cacheDir, "camera_recording_${id}_${System.currentTimeMillis()}.mp4").absolutePath 110 + val slot = RecordingSlot( 111 + builder = null, // Lazily created on first frame 112 + firstTimestampMs = null, 113 + outputPath = outputPath 111 114 ) 112 - recordingActive = true 115 + activeRecordings = activeRecordings + (id to slot) 113 116 } 114 117 115 - suspend fun stopRecording() { 116 - videoBuilder?.let { builder -> 117 - val path = builder.finalize() 118 - videoSavedPath = path 119 - onVideoSaved(path) 120 - videoBuilder = null 121 - firstFrameTimestamp = null 118 + suspend fun stopRecording(id: String) { 119 + activeRecordings[id]?.let { slot -> 120 + slot.builder?.let { builder -> 121 + try { 122 + val path = builder.finalize() 123 + onVideoSaved(id, path) 124 + } catch (e: Exception) { 125 + println("Error finalizing recording $id: ${e.message}") 126 + } 127 + } 128 + activeRecordings = activeRecordings - id 122 129 } 123 - recordingActive = false 124 130 } 125 - fun addFrame(it: ImageBitmap, timestamp: Long) { 126 - if (recordingActive && videoBuilder != null) { 131 + 132 + fun addFrameToActiveRecordings(frame: ImageBitmap, timestampMs: Long) { 133 + if (activeRecordings.isNotEmpty()) { 127 134 scope.launch { 128 - if (firstFrameTimestamp == null) firstFrameTimestamp = timestamp 129 - val relativeTimestamp = timestamp - (firstFrameTimestamp ?: timestamp) 130 - videoBuilder?.addFrame(it, relativeTimestamp) 135 + activeRecordings.forEach { (id, slot) -> 136 + try { 137 + // Lazily create builder on first frame to lock width/height 138 + if (slot.builder == null) { 139 + slot.builder = com.performancecoachlab.posedetection.encoding.createVideoBuilder( 140 + outputPath = slot.outputPath, 141 + fps = 30, 142 + width = frame.width, 143 + height = frame.height 144 + ) 145 + slot.firstTimestampMs = timestampMs 146 + } 147 + 148 + slot.builder?.let { builder -> 149 + val relativeTimestamp = timestampMs - (slot.firstTimestampMs ?: timestampMs) 150 + builder.addFrame(frame, relativeTimestamp) 151 + } 152 + } catch (e: Exception) { 153 + // Handle recording errors gracefully 154 + println("Error recording frame for ID $id: ${e.message}") 155 + } 156 + } 131 157 } 132 158 } 133 159 } 134 160 135 - // React to isRecording changes 136 - LaunchedEffect(isRecording) { 137 - if (isRecording && !recordingActive && bitmap != null) { 138 - // Start recording with current frame size 139 - startRecording(bitmap!!.width, bitmap!!.height) 140 - } else if (!isRecording && recordingActive) { 141 - stopRecording() 161 + // React to recordingIds changes - diff and start/stop recordings 162 + LaunchedEffect(recordingId) { 163 + val currentIds = activeRecordings.keys 164 + recordingId?.also { id -> 165 + currentIds.forEach { 166 + if(!it.contentEquals(id)){ 167 + stopRecording(it) 168 + } 169 + } 170 + if(!currentIds.contains(id)){ 171 + if (bitmap != null) { 172 + startRecording(id, bitmap!!.width, bitmap!!.height) 173 + } else { 174 + // If no frame yet, create empty slot that will be initialized on first frame 175 + val outputPath = File(context.cacheDir, "camera_recording_${id}_${System.currentTimeMillis()}.mp4").absolutePath 176 + val slot = RecordingSlot( 177 + builder = null, 178 + firstTimestampMs = null, 179 + outputPath = outputPath 180 + ) 181 + activeRecordings = activeRecordings + (id to slot) 182 + } 183 + } 184 + }?: run { 185 + // No recordingId - stop all 186 + currentIds.forEach { 187 + stopRecording(it) 188 + } 142 189 } 190 + 191 + 143 192 } 144 193 145 194 LaunchedEffect(lifecycleOwner) { ··· 165 214 bitmap = 166 215 imageProxy.toBitmap().rotate(imageProxy.imageInfo.rotationDegrees.toFloat()) 167 216 .asImageBitmap().let { inbmp -> 168 - addFrame(inbmp,timestamp) 217 + addFrameToActiveRecordings(inbmp, timestamp) 169 218 inbmp.drawResults(if(drawSkeleton) it.skeleton else null, drawObjects?.invoke(it.objects)?: emptyList()) 170 219 } 171 220 imageProxy.close() ··· 180 229 ) 181 230 previewView.scaleType = PreviewView.ScaleType.FIT_CENTER 182 231 } 232 + 183 233 Box( 184 234 modifier = modifier 185 235 ) {

+2 -2

posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.kt

··· 31 31 }, 32 32 modifier: Modifier = Modifier.fillMaxSize(), 33 33 frontCamera: Boolean = true, 34 - isRecording: Boolean = false, 34 + recordingId: String? = null, 35 35 focusArea: Rect? = null, 36 36 onRecordToggled: (Boolean) -> Unit = {}, 37 - onVideoSaved: (String) -> Unit = {}, 37 + onVideoSaved: (String, String) -> Unit 38 38 ) 39 39 40 40 data class DrawableObject(

+6 -8

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt

··· 204 204 cameraController.frameListener = listener 205 205 } 206 206 207 - fun startRecording() { 208 - cameraController.startRecording() 209 - } 207 + fun startRecording() = cameraController.startRecording() 210 208 211 209 fun stopRecording() { 212 210 cameraController.stopRecording() 213 211 } 214 212 215 - fun splitRecording() { 216 - cameraController.splitRecording() 217 - } 213 + fun splitRecording() = cameraController.splitRecording() 218 214 219 215 fun setOnVideoSavedCallback(callback: (String) -> Unit) { 220 216 onVideoSaved = callback ··· 262 258 class CaptureError(message: String) : CameraException() 263 259 } 264 260 265 - fun startRecording() { 261 + fun startRecording(): String? { 266 262 if (movieFileOutput == null) { 267 263 movieFileOutput = AVCaptureMovieFileOutput() 268 264 captureSession?.addOutput(movieFileOutput!!) ··· 277 273 278 274 } 279 275 movieFileOutput?.startRecordingToOutputFileURL(outputURL, this) 276 + return outputURL.path 280 277 } 281 278 282 279 fun stopRecording() { 283 280 movieFileOutput?.stopRecording() 284 281 } 285 282 286 - fun splitRecording() { 283 + fun splitRecording(): String? { 287 284 movieFileOutput?.stopRecording() 288 285 val outputURL = generateSegmentURL() 289 286 movieFileOutput?.startRecordingToOutputFileURL(outputURL, this) 287 + return outputURL.path 290 288 } 291 289 292 290 private fun generateSegmentURL(): platform.Foundation.NSURL {

+25 -7

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt

··· 34 34 drawObjects: ((List<AnalysisObject>) -> List<DrawableObject>)?, 35 35 modifier: Modifier, 36 36 frontCamera: Boolean, 37 - isRecording: Boolean, 37 + recordingId: String?, 38 38 focusArea: Rect?, 39 39 onRecordToggled: (Boolean) -> Unit, 40 - onVideoSaved: (String) -> Unit, 40 + onVideoSaved: (String, String) -> Unit, 41 41 ) { 42 42 val cameraEngine = remember { mutableStateOf<CameraEngine?>(null) } 43 43 val frameListener = remember { FrameRepository() } 44 44 val frameBitmap by frameListener.frameFlow.collectAsState() 45 45 var lastRecordingState by remember { mutableStateOf(false) } 46 + var idMap by remember { mutableStateOf<Map<String, String>>(emptyMap()) } 46 47 LaunchedEffect(detectMode) { 47 48 cameraEngine.value?.setDetectMode(detectMode) 48 49 } 49 - LaunchedEffect(cameraEngine.value, isRecording) { 50 + val recordingDone = {path: String -> 51 + val id = idMap.entries.firstOrNull { it.value == path }?.key 52 + if (id != null) { 53 + onVideoSaved(id, path) 54 + idMap = idMap - id 55 + } 56 + } 57 + LaunchedEffect(cameraEngine.value, recordingId) { 50 58 cameraEngine.value?.apply { 51 59 addSkeletonRepository(skeletonRepository) 52 60 addCustomObjectRepository(customObjectRepository) 53 61 addFrameListener(frameListener) 54 - setOnVideoSavedCallback(onVideoSaved) 62 + setOnVideoSavedCallback(recordingDone) 55 63 setDrawOptions( 56 64 drawSkeleton = drawSkeleton, 57 65 drawObjects = drawObjects, 58 66 ) 59 - if (isRecording && !lastRecordingState) startRecording() 60 - if (!isRecording && lastRecordingState) stopRecording() 61 - lastRecordingState = isRecording 67 + 68 + if (recordingId != null) { 69 + (if(lastRecordingState) splitRecording() 70 + else startRecording())?.also{ 71 + idMap = idMap + (recordingId to it) 72 + lastRecordingState = true 73 + } 74 + } else { 75 + if(lastRecordingState){ 76 + stopRecording() 77 + } 78 + lastRecordingState = false 79 + } 62 80 } 63 81 } 64 82 LaunchedEffect(focusArea){

-191

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/encoding/VideoBuilder.ios.kt

··· 222 222 frameCount = 0L 223 223 } 224 224 225 - @OptIn(ExperimentalForeignApi::class, NativeRuntimeApi::class) 226 - fun buildVideoFromImages( 227 - images: List<ImageBitmap>, 228 - outputSize: CValue<CGSize> = CGSizeMake(width.toDouble(), height.toDouble()) 229 - ): String { 230 - CGSizeMake(1.0, 1.0) 231 - // Create output file path 232 - val fileManager = NSFileManager.defaultManager 233 - 234 - // Delete existing file if needed 235 - if (fileManager.fileExistsAtPath(outputPath)) { 236 - val success = fileManager.removeItemAtPath(outputPath, null) 237 - if (!success) { 238 - throw IllegalStateException("Unable to delete existing file") 239 - } 240 - } 241 - 242 - // Create asset writer 243 - val videoWriter = AVAssetWriter(NSURL.fileURLWithPath(outputPath), AVFileTypeMPEG4, null) 244 - ?: throw IllegalStateException("AVAssetWriter error") 245 - 246 - // Configure video settings 247 - val width = outputSize.useContents { width.toInt() } 248 - val height = outputSize.useContents { height.toInt() } 249 - 250 - val outputSettings = mapOf( 251 - AVVideoCodecKey.toString() to AVVideoCodecH264, 252 - AVVideoWidthKey.toString() to width, 253 - AVVideoHeightKey.toString() to height, 254 - AVVideoCompressionPropertiesKey.toString() to mapOf( 255 - AVVideoAverageBitRateKey.toString() to 6000000, // 6 Mbps 256 - AVVideoProfileLevelKey.toString() to AVVideoProfileLevelH264HighAutoLevel 257 - ) 258 - ) 259 - val sourcePixelBufferAttributes = mapOf( 260 - CFBridgingRelease(kCVPixelBufferPixelFormatTypeKey) as String to kCVPixelFormatType_32BGRA, 261 - CFBridgingRelease(kCVPixelBufferWidthKey) as String to width, 262 - CFBridgingRelease(kCVPixelBufferHeightKey) as String to height, 263 - CFBridgingRelease(kCVPixelBufferCGImageCompatibilityKey) as String to true, 264 - CFBridgingRelease(kCVPixelBufferCGBitmapContextCompatibilityKey) as String to true 265 - ) 266 - val videoWriterInput = AVAssetWriterInput( 267 - mediaType = AVMediaTypeVideo, outputSettings = outputSettings as Map<Any?, *>? 268 - ) 269 - // Add input to writer and start session 270 - if (videoWriter.canAddInput(videoWriterInput)) { 271 - println("[VideoBuilder] Adding input to writer.") 272 - videoWriter.addInput(videoWriterInput) 273 - } else { 274 - println("[VideoBuilder] Cannot add input to writer. canAddInput returned false.") 275 - throw IllegalStateException("Cannot add input to writer") 276 - } 277 - println("[VideoBuilder] videoWriterInput.outputSettings: $outputSettings") 278 - println("[VideoBuilder] sourcePixelBufferAttributes: $sourcePixelBufferAttributes") 279 - println("[VideoBuilder] videoWriter status before startWriting: ${videoWriter.status}") 280 - // Create pixel buffer adaptor AFTER input is added 281 - val pixelBufferAdaptor = AVAssetWriterInputPixelBufferAdaptor( 282 - assetWriterInput = videoWriterInput, 283 - sourcePixelBufferAttributes = sourcePixelBufferAttributes as Map<Any?, *>? 284 - ) 285 - 286 - videoWriter.startWriting() 287 - println("[VideoBuilder] videoWriter status after startWriting: ${videoWriter.status}") 288 - videoWriter.startSessionAtSourceTime(CMTimeMake(0, 1)) 289 - println("[VideoBuilder] videoWriter status after startSessionAtSourceTime: ${videoWriter.status}") 290 - 291 - val fps: Int = 30 292 - val frameDuration = CMTimeMake(1, fps) 293 - var frameCount: Long = 0 294 - val remainingImages = images.toMutableList() 295 - // Only access pixelBufferPool after writing has started 296 - val pool = pixelBufferAdaptor.pixelBufferPool 297 - if (pool == null) { 298 - println("[VideoBuilder] pixelBufferAdaptor.pixelBufferPool is STILL null after startWriting/startSessionAtSourceTime!") 299 - println("[VideoBuilder] videoWriter status: ${videoWriter.status}") 300 - println("[VideoBuilder] videoWriterInput isReadyForMoreMediaData: ${videoWriterInput.readyForMoreMediaData}") 301 - throw IllegalStateException("Pixel buffer pool is null after startWriting/startSessionAtSourceTime") 302 - } 303 - 304 - // Process each image 305 - videoWriterInput.requestMediaDataWhenReadyOnQueue( 306 - dispatch_queue_create( 307 - "mediaInputQueue", null 308 - ) 309 - ) { 310 - var appendSucceeded = true 311 - 312 - while (remainingImages.isNotEmpty() && appendSucceeded) { 313 - if (videoWriterInput.readyForMoreMediaData) { 314 - val nextPhoto = remainingImages.removeAt(0) 315 - val lastFrameTime = CMTimeMake(frameCount, fps) 316 - val presentationTime = if (frameCount == 0L) lastFrameTime else CMTimeAdd( 317 - lastFrameTime, frameDuration 318 - ) 319 - 320 - memScoped { 321 - val pixelBufferPtr = alloc<CVPixelBufferRefVar>() 322 - val status = CVPixelBufferPoolCreatePixelBuffer( 323 - kCFAllocatorDefault, pool, pixelBufferPtr.ptr 324 - ) 325 - 326 - if (status == kCVReturnSuccess) { 327 - val pixelBuffer = pixelBufferPtr.value 328 - CVPixelBufferLockBaseAddress(pixelBuffer, 0u) 329 - 330 - val baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer) 331 - val bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer) 332 - val colorSpace = CGColorSpaceCreateDeviceRGB() 333 - val bitmapInfo = CGImageAlphaInfo.kCGImageAlphaPremultipliedFirst.value 334 - 335 - val context = CGBitmapContextCreate( 336 - baseAddress, 337 - width.convert(), 338 - height.convert(), 339 - 8u, 340 - bytesPerRow.convert(), 341 - colorSpace, 342 - bitmapInfo 343 - ) 344 - 345 - if (context != null) { 346 - // Clear context 347 - CGContextClearRect( 348 - context, 349 - CGRectMake(0.0, 0.0, width.toDouble(), height.toDouble()) 350 - ) 351 - 352 - // Use ImageBitmap pixel data directly 353 - println("image processed") 354 - val pixelMap = nextPhoto.toPixelMap() 355 - val buffer = ensureFourChannelBuffer(pixelMap) 356 - val contextData = baseAddress?.reinterpret<ByteVar>() 357 - // Debug prints 358 - println("[VideoBuilder] ensureFourChannelBuffer: buffer.size=${buffer.size}, expected=${width * height * 4}") 359 - println("[VideoBuilder] CGContext bytesPerRow: $bytesPerRow, expected=${width * 4}") 360 - if (buffer.size != width * height * 4) { 361 - println("[VideoBuilder] ERROR: Buffer size does not match expected size for BGRA data!") 362 - } 363 - if (contextData != null) { 364 - // Copy row by row to handle bytesPerRow alignment 365 - val srcRowBytes = width * 4 366 - val dstRowBytes = bytesPerRow.toInt() 367 - for (row in 0 until height) { 368 - buffer.usePinned { pinned -> 369 - val srcOffset = row * srcRowBytes 370 - val dstOffset = row * dstRowBytes 371 - val dstPtr = contextData.plus(dstOffset) 372 - platform.posix.memcpy( 373 - dstPtr, 374 - pinned.addressOf(srcOffset), 375 - srcRowBytes.toULong() 376 - ) 377 - } 378 - } 379 - } 380 - CGContextRelease(context) 381 - } 382 - 383 - CGColorSpaceRelease(colorSpace) 384 - CVPixelBufferUnlockBaseAddress(pixelBuffer, 0u) 385 - 386 - // Append frame to video 387 - appendSucceeded = pixelBufferAdaptor.appendPixelBuffer( 388 - pixelBuffer, withPresentationTime = presentationTime 389 - ) 390 - CVPixelBufferRelease(pixelBuffer) 391 - } else { 392 - println("Failed to allocate pixel buffer: $status") 393 - appendSucceeded = false 394 - } 395 - } 396 - 397 - frameCount++ 398 - } 399 - } 400 - 401 - videoWriterInput.markAsFinished() 402 - videoWriter.finishWritingWithCompletionHandler { 403 - println("[VideoBuilder] Finished writing video!") 404 - } 405 - } 406 - 407 - // Wait for writing to finish (this is simplified) 408 - // In a real app, you would handle this asynchronously 409 - while (videoWriter.status == AVAssetWriterStatusWriting) { 410 - // Wait for completion 411 - } 412 - 413 - return outputPath 414 - } 415 - 416 225 private fun ensureFourChannelBuffer(pixelMap: PixelMap): ByteArray { 417 226 val width = pixelMap.width 418 227 val height = pixelMap.height

+9 -6

sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt

··· 63 63 import kotlinx.coroutines.Job 64 64 import kotlinx.coroutines.launch 65 65 import kotlin.math.roundToLong 66 + import kotlin.time.Clock 67 + import kotlin.time.ExperimentalTime 66 68 67 69 @Composable 68 70 internal fun App() = AppTheme { ··· 276 278 } 277 279 } 278 280 281 + @OptIn(ExperimentalTime::class) 279 282 @Composable 280 283 fun CameraSample() { 281 284 val skeletonRepository = remember { SkeletonRepository() } ··· 283 286 val skeleton by skeletonRepository.skeletonFlow.collectAsState() 284 287 val customObjects by customObjectRespository.customObjectFlow.collectAsState() 285 288 var permissionGranted by remember { mutableStateOf(false) } 286 - var isRecording by remember { mutableStateOf(false) } 289 + var recordingId : String? by remember { mutableStateOf(null) } 287 290 var path by remember { mutableStateOf("") } 288 291 val generalModel = initialiseObjectModel( 289 292 ModelPath( ··· 328 331 CameraView( 329 332 skeletonRepository = skeletonRepository, 330 333 customObjectRepository = customObjectRespository, 334 + detectMode = DetectMode.NONE, 331 335 drawSkeleton = true, 332 336 objectModel = generalModel, 333 337 modifier = Modifier.weight(1f), 334 338 frontCamera = true, 335 - isRecording = isRecording, 336 - onRecordToggled = { isRecording = it }, 337 - onVideoSaved = { path = it }, 339 + recordingId = recordingId, 340 + onVideoSaved = {id,url -> path = url }, 338 341 ) 339 342 } 340 343 Button( 341 344 onClick = { 342 - isRecording = !isRecording 345 + recordingId = "${Clock.System.now().epochSeconds}" 343 346 }, 344 347 modifier = Modifier.imePadding().padding(16.dp).align(Alignment.TopStart) 345 348 ) { 346 - Text(if (isRecording) "Stop Recording" else "Start Recording") 349 + Text("Start Recording") 347 350 } 348 351 } 349 352 } else Text("Camera permission not granted")

Configure Feed

Configure Feed