+105 -79

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt

Reviewed

··· 221 221 cameraController.switchCamera() 222 222 } 223 223 224 224 + /** 225 225 + * Deterministically select which lens to use. 226 226 + * This is idempotent (won't switch if already on the requested lens), 227 227 + * which makes it safe to call from Compose recomposition. 228 228 + */ 229 229 + fun setUseFrontCamera(useFront: Boolean) { 230 230 + MemoryManager.updateMemoryStatus() 231 231 + if (MemoryManager.isUnderMemoryPressure()) MemoryManager.clearBufferPools() 232 232 + cameraController.setUseFrontCamera(useFront) 233 233 + } 234 234 + 224 235 fun startSession() { 225 236 MemoryManager.clearBufferPools() 226 237 cameraController.startSession() ··· 288 299 private var videoOutput: AVCaptureVideoDataOutput? = null 289 300 private var movieFileOutput: AVCaptureMovieFileOutput? = null 290 301 var cameraPreviewLayer: AVCaptureVideoPreviewLayer? = null 291 291 - var isUsingFrontCamera = true 302 302 + var isUsingFrontCamera = false 292 303 var skeletonRepository: SkeletonRepository? = null 293 304 var customObjectRepository: CustomObjectRespository? = null 294 305 var cameraViewController: CameraViewController? = null ··· 302 313 303 314 // Serial queue to serialize session configuration and start/stop calls 304 315 private val sessionQueue = dispatch_queue_create("com.performancecoachlab.captureSessionQueue", null) 316 316 + 317 317 + // Prevent overlapping switch operations (which can deadlock/jank the UI). 318 318 + private var isSwitchingCamera: Boolean = false 319 319 + 320 320 + // Reuse a single queue for frame processing work. 321 321 + private val frameProcessingQueue = dispatch_queue_create("com.performancecoachlab.frameProcessing", null) 305 322 306 323 sealed class CameraException : Exception() { 307 324 class DeviceNotAvailable : CameraException() ··· 365 382 frameProcessor.setObjectModel(objectModel) 366 383 } 367 384 385 385 + /** 386 386 + * Deterministically select which lens to use. 387 387 + * No-ops if the requested lens is already active. 388 388 + */ 389 389 + fun setUseFrontCamera(useFront: Boolean) { 390 390 + if (isUsingFrontCamera == useFront) return 391 391 + switchCamera() 392 392 + } 393 393 + 368 394 fun setupSession() { 369 395 // Run synchronously on sessionQueue so callers (like setupCamera) can safely call startSession afterwards 370 396 dispatch_sync(sessionQueue) { ··· 406 432 407 433 private fun setupVideoOutput() { 408 434 videoOutput = AVCaptureVideoDataOutput() 409 409 - videoOutput?.setSampleBufferDelegate(this, dispatch_get_main_queue()) 435 435 + // Deliver frames off the main thread; toggling/config and UI should not compete with capture callbacks. 436 436 + videoOutput?.setSampleBufferDelegate(this, sessionQueue) 410 437 videoOutput?.videoSettings = mapOf(AVVideoCodecKey to AVVideoCodecJPEG) 411 438 412 439 if (captureSession?.canAddOutput(videoOutput!!) == true) { ··· 516 543 fun switchCamera() { 517 544 guard(captureSession != null) { return@guard } 518 545 519 519 - // Run camera switch on sessionQueue to serialize with setupSession/startSession 520 520 - dispatch_sync(sessionQueue) { 546 546 + // Never block the caller thread (often the main thread). Serialize on sessionQueue. 547 547 + dispatch_async(sessionQueue) { 548 548 + if (isSwitchingCamera) return@dispatch_async 549 549 + isSwitchingCamera = true 550 550 + 521 551 var configurationBegan = false 522 552 try { 523 553 captureSession?.beginConfiguration() ··· 533 563 val newCamera = currentCamera ?: throw CameraException.DeviceNotAvailable() 534 564 535 565 val newInput = AVCaptureDeviceInput.deviceInputWithDevice( 536 536 - newCamera, null 566 566 + newCamera, 567 567 + null 537 568 ) ?: throw CameraException.ConfigurationError("Failed to create input") 538 569 539 570 if (captureSession?.canAddInput(newInput) == true) { ··· 548 579 connection.setVideoMirrored(isUsingFrontCamera) 549 580 } 550 581 } 551 551 - 552 582 } catch (e: CameraException) { 553 583 onError?.invoke(e) 554 584 } catch (e: Exception) { ··· 557 587 if (configurationBegan) { 558 588 captureSession?.commitConfiguration() 559 589 } 590 590 + isSwitchingCamera = false 560 591 } 561 592 } 562 593 } ··· 571 602 ) { 572 603 timeStamp().also { timestamp -> 573 604 runCatching { 574 574 - dispatch_queue_create( 575 575 - "com.performancecoachlab.frameProcessing", null 576 576 - ).also { processingQueue -> 577 577 - dispatch_async(processingQueue) { 578 578 - try { 579 579 - var detectedSkeleton: Skeleton? = null 580 580 - var detectedObjects: List<AnalysisObject> = emptyList() 581 581 - frameProcessor.analyseBufferForAll( 582 582 - CMSampleBufferGetImageBuffer(didOutputSampleBuffer), 583 583 - timestamp, 584 584 - preview = cameraPreviewLayer, 585 585 - onSkeletonProcessed = { skeleton -> 586 586 - detectedSkeleton = skeleton 587 587 - }, 588 588 - onObjectsProcessed = { objects -> 589 589 - detectedObjects = objects 590 590 - }) 591 591 - cameraPreviewLayer?.also { preview -> 592 592 - val previewSkeleton = detectedSkeleton?.let { 593 593 - mapSkeletonToPreview( 594 594 - skeleton = it, 595 595 - previewLayer = preview, 605 605 + dispatch_async(frameProcessingQueue) { 606 606 + try { 607 607 + var detectedSkeleton: Skeleton? = null 608 608 + var detectedObjects: List<AnalysisObject> = emptyList() 609 609 + frameProcessor.analyseBufferForAll( 610 610 + CMSampleBufferGetImageBuffer(didOutputSampleBuffer), 611 611 + timestamp, 612 612 + preview = cameraPreviewLayer, 613 613 + onSkeletonProcessed = { skeleton -> 614 614 + detectedSkeleton = skeleton 615 615 + }, 616 616 + onObjectsProcessed = { objects -> 617 617 + detectedObjects = objects 618 618 + } 619 619 + ) 620 620 + cameraPreviewLayer?.also { preview -> 621 621 + val previewSkeleton = detectedSkeleton?.let { 622 622 + mapSkeletonToPreview( 623 623 + skeleton = it, 624 624 + previewLayer = preview, 625 625 + width = 480f, 626 626 + height = 360f 627 627 + ) 628 628 + } 629 629 + previewSkeleton?.also { 630 630 + skeletonRepository?.updateSkeleton(it) 631 631 + } 632 632 + val previewObjects = detectedObjects.map { 633 633 + it.copy( 634 634 + boundingBox = mapBoxToPreview( 635 635 + it.boundingBox, 636 636 + preview, 596 637 width = 480f, 597 638 height = 360f 598 598 - ) 599 599 - } 600 600 - previewSkeleton?.also { 601 601 - skeletonRepository?.updateSkeleton( 602 602 - it 603 603 - ) 604 604 - } 605 605 - val previewObjects = detectedObjects.map { 606 606 - it.copy( 607 607 - boundingBox = mapBoxToPreview( 608 608 - it.boundingBox, preview, width = 480f, height = 360f 609 609 - ), frameSize = it.frameSize.let { 610 610 - mapBoxToPreview( 611 611 - Rect(Offset.Zero, Size(480f, 360f)), 612 612 - preview, 613 613 - width = 480f, 614 614 - height = 360f 615 615 - ).let { FrameSize(it.width.toInt().absoluteValue, it.height.toInt().absoluteValue) } 616 616 - }) 617 617 - } 618 618 - previewObjects.also { objects -> 619 619 - customObjectRepository?.updateCustomObject(objects) 620 620 - } 621 621 - preview.bounds.useContents { 622 622 - Pair( 623 623 - size.width.toInt(), size.height.toInt() 624 624 - ) 625 625 - }.also { bo -> 626 626 - ImageBitmap( 627 627 - bo.first, bo.second 628 628 - ).drawResults( 629 629 - if (drawSkeleton) previewSkeleton else null, 630 630 - drawObjects?.invoke(previewObjects) ?: emptyList(), 631 631 - textMeasurer = textMeasurer 632 632 - ).also { drawn -> 633 633 - cameraViewController?.setRequestDataProvider { 634 634 - CameraViewData( 635 635 - width = bo.first.toFloat(), 636 636 - height = bo.second.toFloat(), 637 637 - rotation = when (currentVideoOrientation()) { 638 638 - AVCaptureVideoOrientationPortrait -> SensorRotation.ROTATION_90 639 639 - AVCaptureVideoOrientationLandscapeRight -> SensorRotation.ROTATION_0 640 640 - AVCaptureVideoOrientationPortraitUpsideDown -> SensorRotation.ROTATION_270 641 641 - AVCaptureVideoOrientationLandscapeLeft -> SensorRotation.ROTATION_180 642 642 - else -> SensorRotation.ROTATION_0 643 643 - } 644 644 - ) 639 639 + ), 640 640 + frameSize = it.frameSize.let { 641 641 + mapBoxToPreview( 642 642 + Rect(Offset.Zero, Size(480f, 360f)), 643 643 + preview, 644 644 + width = 480f, 645 645 + height = 360f 646 646 + ).let { r -> 647 647 + FrameSize(r.width.toInt().absoluteValue, r.height.toInt().absoluteValue) 645 648 } 646 646 - frameListener?.updateFrame( 647 647 - drawn 649 649 + } 650 650 + ) 651 651 + } 652 652 + customObjectRepository?.updateCustomObject(previewObjects) 653 653 + 654 654 + preview.bounds.useContents { 655 655 + Pair(size.width.toInt(), size.height.toInt()) 656 656 + }.also { bo -> 657 657 + ImageBitmap(bo.first, bo.second).drawResults( 658 658 + if (drawSkeleton) previewSkeleton else null, 659 659 + drawObjects?.invoke(previewObjects) ?: emptyList(), 660 660 + textMeasurer = textMeasurer 661 661 + ).also { drawn -> 662 662 + cameraViewController?.setRequestDataProvider { 663 663 + CameraViewData( 664 664 + width = bo.first.toFloat(), 665 665 + height = bo.second.toFloat(), 666 666 + rotation = when (currentVideoOrientation()) { 667 667 + AVCaptureVideoOrientationPortrait -> SensorRotation.ROTATION_90 668 668 + AVCaptureVideoOrientationLandscapeRight -> SensorRotation.ROTATION_0 669 669 + AVCaptureVideoOrientationPortraitUpsideDown -> SensorRotation.ROTATION_270 670 670 + AVCaptureVideoOrientationLandscapeLeft -> SensorRotation.ROTATION_180 671 671 + else -> SensorRotation.ROTATION_0 672 672 + } 648 673 ) 649 674 } 675 675 + frameListener?.updateFrame(drawn) 650 676 } 651 677 } 652 652 - } catch (e: Exception) { 653 653 - // ignore frame processing errors 654 678 } 679 679 + } catch (_: Exception) { 680 680 + // ignore frame processing errors 655 681 } 656 682 } 657 683 }

+13 -3

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt

Reviewed

··· 50 50 LaunchedEffect(detectMode) { 51 51 cameraEngine.value?.setDetectMode(detectMode) 52 52 } 53 53 + 54 54 + // Ensure lens selection is deterministic and recomposition-safe. 55 55 + LaunchedEffect(frontCamera) { 56 56 + cameraEngine.value?.setUseFrontCamera(frontCamera) 57 57 + } 58 58 + 53 59 val recordingDone = {path: String -> 54 60 val id = idMap.entries.firstOrNull { it.value == path }?.key 55 61 if (id != null) { ··· 98 104 } 99 105 Box(modifier = Modifier.fillMaxSize()) { 100 106 CameraPreview( 101 101 - modifier = Modifier.fillMaxSize(), onCameraControllerReady = { engine -> 102 102 - cameraEngine.value = engine.also { if (!frontCamera) it.toggleCameraLens() } 103 103 - },) 107 107 + modifier = Modifier.fillMaxSize(), 108 108 + onCameraControllerReady = { engine -> 109 109 + cameraEngine.value = engine 110 110 + // Apply immediately as well for the first composition. 111 111 + engine.setUseFrontCamera(frontCamera) 112 112 + }, 113 113 + ) 104 114 if (drawSkeleton || drawObjects!= null) { 105 115 kotlin.native.runtime.GC.collect() 106 116 frameBitmap?.also {

+3 -5

sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt

Reviewed

··· 316 316 "YOLOv3FP16" 317 317 ) 318 318 ) 319 319 + var frontCamera by remember { mutableStateOf(true) } 319 320 val controller = remember { CameraViewControllerImpl() } 320 321 PermissionProvider().apply { 321 322 if (!hasCameraPermission()) RequestCameraPermission(onGranted = { ··· 369 370 objectModel = generalModel, 370 371 modifier = Modifier.weight(1f), 371 372 focusArea = Rect(0f,0f,0.1f,1f), 372 372 - frontCamera = false, 373 373 + frontCamera = frontCamera, 373 374 recordingId = recordingId, 374 375 controller = controller, 375 376 onVideoSaved = {id,url -> path = url }, ··· 377 378 } 378 379 Button( 379 380 onClick = { 380 380 - //recordingId = "${Clock.System.now().epochSeconds}" 381 381 - controller.requestData { data -> 382 382 - Logger.d("CameraViewData: $data") 383 383 - } 381 381 + frontCamera = !frontCamera 384 382 }, 385 383 modifier = Modifier.imePadding().padding(16.dp).align(Alignment.TopStart) 386 384 ) {