This repository has no description
feat: control overlays for objects and skeleton separately
author
nathan holland
date
(Jul 20, 2025, 10:38 PM +0100)
commit
607dc9d2
607dc9d29a39cb75fb38fb7056da04d4e586d762
parent
44f42b75
44f42b754c44eecd70c3560cf9ed8866e2595612
+71
-44
+5
-3
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
+5
-3
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
···
61
61
skeletonRepository: SkeletonRepository,
62
62
customObjectRepository: CustomObjectRespository,
63
63
drawSkeleton: Boolean,
64
+
drawObjects: Boolean,
64
65
modifier: Modifier,
65
66
frontCamera: Boolean,
66
67
isRecording: Boolean,
···
146
147
imageProxy.toBitmap().rotate(imageProxy.imageInfo.rotationDegrees.toFloat())
147
148
.asImageBitmap().let { inbmp ->
148
149
addFrame(inbmp,timestamp)
149
-
if( drawSkeleton) {
150
+
AnalysisResult(
151
+
skeleton = if(drawSkeleton) it.skeleton else null,
152
+
objects = if(drawObjects) it.objects else emptyList(),
153
+
).let{
150
154
inbmp.drawAnalysisResults(it)
151
-
}else{
152
-
inbmp.drawSkeleton(null)
153
155
}
154
156
}
155
157
imageProxy.close()
+1
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.kt
+1
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.kt
+57
-39
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
+57
-39
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
···
216
216
onVideoSaved = callback
217
217
cameraController.onVideoSaved = callback
218
218
}
219
+
220
+
fun setDrawOptions(drawSkeleton: Boolean, drawObjects: Boolean) {
221
+
cameraController.drawSkeleton = drawSkeleton
222
+
cameraController.drawObjects = drawObjects
223
+
}
219
224
}
220
225
221
-
class CameraController : NSObject(), AVCaptureVideoDataOutputSampleBufferDelegateProtocol,platform.AVFoundation.AVCaptureFileOutputRecordingDelegateProtocol {
226
+
class CameraController : NSObject(), AVCaptureVideoDataOutputSampleBufferDelegateProtocol,
227
+
platform.AVFoundation.AVCaptureFileOutputRecordingDelegateProtocol {
222
228
private var captureSession: AVCaptureSession? = null
223
229
private var backCamera: AVCaptureDevice? = null
224
230
private var frontCamera: AVCaptureDevice? = null
···
228
234
var cameraPreviewLayer: AVCaptureVideoPreviewLayer? = null
229
235
var isUsingFrontCamera = true
230
236
var skeletonRepository: SkeletonRepository? = null
231
-
var customObjectRepository:CustomObjectRespository? = null
237
+
var customObjectRepository: CustomObjectRespository? = null
232
238
var frameListener: FrameRepository? = null
233
239
var onError: ((CameraException) -> Unit)? = null
234
240
var startTime: Long? = null
235
241
var onVideoSaved: ((String) -> Unit)? = null
242
+
var drawSkeleton: Boolean = true
243
+
var drawObjects: Boolean = true
236
244
237
245
sealed class CameraException : Exception() {
238
246
class DeviceNotAvailable : CameraException()
239
247
class ConfigurationError(message: String) : CameraException()
240
248
class CaptureError(message: String) : CameraException()
241
249
}
250
+
242
251
fun startRecording() {
243
252
if (movieFileOutput == null) {
244
253
movieFileOutput = AVCaptureMovieFileOutput()
245
254
captureSession?.addOutput(movieFileOutput!!)
246
255
}
247
256
val outputURL = generateSegmentURL()
248
-
movieFileOutput?.connections?.firstOrNull()?.let { connection ->
257
+
movieFileOutput?.connections?.firstOrNull()?.let { connection ->
249
258
(connection as AVCaptureConnection).let { avConnection ->
250
259
if (avConnection.isVideoOrientationSupported()) {
251
260
avConnection.videoOrientation = currentVideoOrientation()
···
269
278
private fun generateSegmentURL(): platform.Foundation.NSURL {
270
279
val tempDir = platform.Foundation.NSTemporaryDirectory()
271
280
val fileName = platform.Foundation.NSUUID().UUIDString + ".mov"
272
-
val filePath = (tempDir as platform.Foundation.NSString).stringByAppendingPathComponent(fileName)
281
+
val filePath =
282
+
(tempDir as platform.Foundation.NSString).stringByAppendingPathComponent(fileName)
273
283
return platform.Foundation.NSURL.fileURLWithPath(filePath)
274
284
}
285
+
275
286
fun setupSession() {
276
287
try {
277
288
captureSession = AVCaptureSession()
···
448
459
didOutputSampleBuffer: CMSampleBufferRef?,
449
460
fromConnection: AVCaptureConnection
450
461
) {
451
-
timeStamp().also { timestamp->
462
+
timeStamp().also { timestamp ->
452
463
runCatching {
453
464
dispatch_queue_create(
454
465
"com.performancecoachlab.frameProcessing", null
···
458
469
var detectedSkeleton: Skeleton? = null
459
470
var detectedObjects: List<AnalysisObject> = emptyList()
460
471
frameProcessor.analyseBufferForAll(
461
-
CMSampleBufferGetImageBuffer(didOutputSampleBuffer), timestamp,
472
+
CMSampleBufferGetImageBuffer(didOutputSampleBuffer),
473
+
timestamp,
462
474
onSkeletonProcessed = { skeleton ->
463
475
skeleton?.also {
464
476
skeletonRepository?.updateSkeleton(
···
470
482
onObjectsProcessed = { objects ->
471
483
detectedObjects = objects
472
484
customObjectRepository?.updateCustomObject(objects)
473
-
}
474
-
)
485
+
})
475
486
cameraPreviewLayer?.also { preview ->
476
487
val previewSkeleton = detectedSkeleton?.let {
477
488
mapSkeletonToPreview(
···
482
493
)
483
494
}
484
495
val previewObjects = detectedObjects.map {
485
-
it.copy(boundingBox = mapBoxToPreview(it.boundingBox,preview,
486
-
width = 480f,
487
-
height = 360f))
496
+
it.copy(
497
+
boundingBox = mapBoxToPreview(
498
+
it.boundingBox, preview, width = 480f, height = 360f
499
+
)
500
+
)
488
501
}
489
502
preview.bounds.useContents {
490
503
Pair(
491
-
size.width.toInt(),
492
-
size.height.toInt()
504
+
size.width.toInt(), size.height.toInt()
493
505
)
494
506
}.also { bo ->
495
507
ImageBitmap(
496
508
bo.first, bo.second
497
-
).drawAnalysisResults(analysisResults = AnalysisResult(skeleton = previewSkeleton, objects = previewObjects))
498
-
.also { drawn ->
499
-
frameListener?.updateFrame(
500
-
drawn
501
-
)
502
-
}
509
+
).drawAnalysisResults(
510
+
analysisResults = AnalysisResult(
511
+
skeleton = if (drawSkeleton) previewSkeleton else null,
512
+
objects = if (drawObjects) previewObjects else emptyList()
513
+
)
514
+
).also { drawn ->
515
+
frameListener?.updateFrame(
516
+
drawn
517
+
)
518
+
}
503
519
}
504
520
}
505
521
} catch (e: Exception) {
···
511
527
}
512
528
MemoryManager.updateMemoryStatus()
513
529
kotlin.native.runtime.GC.collect()
514
-
}
530
+
}
531
+
515
532
fun ImageBitmap.copy(): ImageBitmap {
516
533
val original = this
517
534
val copied = ImageBitmap(width, height, config, hasAlpha, colorSpace)
···
519
536
val drawScope = CanvasDrawScope()
520
537
val size = Size(width.toFloat(), height.toFloat())
521
538
drawScope.draw(
522
-
Density(1f),
523
-
LayoutDirection.Ltr,
524
-
canvas,
525
-
size
539
+
Density(1f), LayoutDirection.Ltr, canvas, size
526
540
) {
527
541
drawImage(original)
528
542
}
529
543
return copied
530
544
}
545
+
531
546
private inline fun guard(condition: Boolean, crossinline block: () -> Unit) {
532
547
if (!condition) block()
533
548
}
···
550
565
height: Float,
551
566
): Rect {
552
567
fun mapPoint(point: Offset): Offset {
553
-
val normalizedPoint = CGPointMake(point.x.toDouble()/width, point.y.toDouble()/height)
568
+
val normalizedPoint =
569
+
CGPointMake(point.x.toDouble() / width, point.y.toDouble() / height)
554
570
val screenPoint = previewLayer.pointForCaptureDevicePointOfInterest(normalizedPoint)
555
-
return Offset(screenPoint.useContents { x.toFloat() }, screenPoint.useContents { y.toFloat() })
571
+
return Offset(
572
+
screenPoint.useContents { x.toFloat() },
573
+
screenPoint.useContents { y.toFloat() })
556
574
}
575
+
557
576
val topLeft = mapPoint(box.topLeft)
558
577
val bottomRight = mapPoint(box.bottomRight)
559
578
return Rect(topLeft = topLeft, bottomRight = bottomRight)
···
561
580
562
581
@OptIn(ExperimentalForeignApi::class)
563
582
fun mapSkeletonToPreview(
564
-
skeleton: Skeleton,
565
-
previewLayer: AVCaptureVideoPreviewLayer,
566
-
width: Float,
567
-
height: Float
583
+
skeleton: Skeleton, previewLayer: AVCaptureVideoPreviewLayer, width: Float, height: Float
568
584
): Skeleton {
569
585
fun mapPoint(point: Skeleton.SkeletonCoordinate?): Skeleton.SkeletonCoordinate? {
570
586
if (point == null) return null
571
587
572
588
// Normalize the point
573
-
val normalizedPoint = CGPointMake(point.x.toDouble()/width, point.y.toDouble()/height)
589
+
val normalizedPoint =
590
+
CGPointMake(point.x.toDouble() / width, point.y.toDouble() / height)
574
591
val screenPoint = previewLayer.pointForCaptureDevicePointOfInterest(normalizedPoint)
575
-
return Skeleton.SkeletonCoordinate(screenPoint.useContents { x.toFloat() }, screenPoint.useContents { y.toFloat() })
592
+
return Skeleton.SkeletonCoordinate(
593
+
screenPoint.useContents { x.toFloat() },
594
+
screenPoint.useContents { y.toFloat() })
576
595
}
577
596
578
597
return Skeleton(
···
619
638
val length = CFDataGetLength(data)
620
639
621
640
val alphaType = when (CGImageGetAlphaInfo(imageRef)) {
622
-
CGImageAlphaInfo.kCGImageAlphaPremultipliedFirst,
623
-
CGImageAlphaInfo.kCGImageAlphaPremultipliedLast -> ColorAlphaType.PREMUL
624
-
CGImageAlphaInfo.kCGImageAlphaFirst,
625
-
CGImageAlphaInfo.kCGImageAlphaLast -> ColorAlphaType.UNPREMUL
626
-
CGImageAlphaInfo.kCGImageAlphaNone,
627
-
CGImageAlphaInfo.kCGImageAlphaNoneSkipFirst,
628
-
CGImageAlphaInfo.kCGImageAlphaNoneSkipLast -> ColorAlphaType.OPAQUE
641
+
CGImageAlphaInfo.kCGImageAlphaPremultipliedFirst, CGImageAlphaInfo.kCGImageAlphaPremultipliedLast -> ColorAlphaType.PREMUL
642
+
643
+
CGImageAlphaInfo.kCGImageAlphaFirst, CGImageAlphaInfo.kCGImageAlphaLast -> ColorAlphaType.UNPREMUL
644
+
645
+
CGImageAlphaInfo.kCGImageAlphaNone, CGImageAlphaInfo.kCGImageAlphaNoneSkipFirst, CGImageAlphaInfo.kCGImageAlphaNoneSkipLast -> ColorAlphaType.OPAQUE
646
+
629
647
else -> ColorAlphaType.UNKNOWN
630
648
}
631
649
+6
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt
+6
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt
···
24
24
skeletonRepository: SkeletonRepository,
25
25
customObjectRepository:CustomObjectRespository,
26
26
drawSkeleton: Boolean,
27
+
drawObjects: Boolean,
27
28
modifier: Modifier,
28
29
frontCamera: Boolean,
29
30
isRecording: Boolean,
···
40
41
addCustomObjectRepository(customObjectRepository)
41
42
addFrameListener(frameListener)
42
43
setOnVideoSavedCallback(onVideoSaved)
44
+
setDrawOptions(
45
+
drawSkeleton = drawSkeleton,
46
+
drawObjects = drawObjects,
47
+
)
43
48
if (isRecording && !lastRecordingState) startRecording()
44
49
if (!isRecording && lastRecordingState) stopRecording()
45
50
lastRecordingState = isRecording
···
50
55
modifier = Modifier.fillMaxSize(), onCameraControllerReady = { engine ->
51
56
cameraEngine.value = engine.also { if (!frontCamera) it.toggleCameraLens() }
52
57
})
53
-
if (drawSkeleton) {
58
+
if (drawSkeleton || drawObjects) {
54
59
kotlin.native.runtime.GC.collect()
55
60
frameBitmap?.also {
56
61
Image(
+2
-1
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
+2
-1
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
···
327
327
CameraView(
328
328
skeletonRepository = skeletonRepository,
329
329
customObjectRepository = customObjectRespository,
330
-
drawSkeleton = true,
330
+
drawSkeleton = false,
331
+
drawObjects = true,
331
332
modifier = Modifier.weight(1f),
332
333
frontCamera = false,
333
334
isRecording = isRecording,