This repository has no description
fix: better tracking in ios
author
nathan holland
date
(Jul 5, 2025, 1:08 PM +0300)
commit
e888d1f6
e888d1f67b24a285dd47a175faf5baa448c4518c
parent
b0e13227
b0e13227204fb241a66b88a124d557a439417290
+147
-5
+139
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
+139
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
···
424
424
}
425
425
426
426
@OptIn(BetaInteropApi::class, ExperimentalNativeApi::class)
427
+
fun analyseFrameForAll(
428
+
cgImage: CGImageRef?,
429
+
timestamp: Long,
430
+
onObjectsProcessed: (List<AnalysisObject>) -> Unit,
431
+
onSkeletonProcessed: (Skeleton?) -> Unit
432
+
){
433
+
autoreleasepool {
434
+
if (cgImage == null) {
435
+
onObjectsProcessed(emptyList())
436
+
onSkeletonProcessed(null)
437
+
return
438
+
}
439
+
val width = CGImageGetWidth(cgImage)
440
+
val height = CGImageGetHeight(cgImage)
441
+
if (width.toUInt() == 0u || height.toUInt() == 0u) {
442
+
onObjectsProcessed(emptyList())
443
+
onSkeletonProcessed(null)
444
+
return
445
+
}
446
+
memScoped {
447
+
val errorPtr = alloc<ObjCObjectVar<NSError?>>()
448
+
if (modelObj == null) {
449
+
onObjectsProcessed(emptyList())
450
+
onSkeletonProcessed(null)
451
+
return@memScoped
452
+
}
453
+
try {
454
+
val requestForObjects = VNCoreMLRequest(modelObj) { request, error ->
455
+
if (error != null) {
456
+
onObjectsProcessed(emptyList())
457
+
return@VNCoreMLRequest
458
+
}
459
+
val results = request?.results as? List<*> ?: emptyList<Any>()
460
+
val recognized = results.filterIsInstance<VNRecognizedObjectObservation>()
461
+
val analysisObjects = recognized.map { observation ->
462
+
val confidence = observation.confidence
463
+
val boundingBox = observation.boundingBox.useContents {
464
+
val w = width.toFloat()
465
+
val h = height.toFloat()
466
+
val left = origin.x * w
467
+
val top = (1.0 - origin.y) * h
468
+
val right = (origin.x + size.width) * w
469
+
val bottom = (1.0 - (origin.y + size.height)) * h
470
+
Rect(
471
+
left = left.toFloat(),
472
+
top = top.toFloat(),
473
+
right = right.toFloat(),
474
+
bottom = bottom.toFloat()
475
+
)
476
+
}
477
+
val labels = observation.labels.mapNotNull {
478
+
(it as VNClassificationObservation).let { ca ->
479
+
if (ca.confidence > 0.0) ca.identifier else null
480
+
}
481
+
}
482
+
AnalysisObject(
483
+
trackingId = observation.identityHashCode(),
484
+
labels = labels,
485
+
boundingBox = boundingBox,
486
+
)
487
+
}
488
+
onObjectsProcessed(analysisObjects)
489
+
}
490
+
val options = mapOf<Any?, Any?>(
491
+
"orientation" to AVCaptureVideoOrientationLandscapeRight
492
+
)
493
+
val requestForSkeleton = VNDetectHumanBodyPoseRequest { request, error ->
494
+
if (error != null) {
495
+
onSkeletonProcessed(null)
496
+
} else {
497
+
request?.also { vnRequest ->
498
+
val recognizedPoints = bodyPoseHandler(vnRequest)
499
+
regionOfInterest = calculateRegionOfInterest(recognizedPoints)
500
+
val updatedSkeleton = Skeleton(
501
+
timestamp = timestamp,
502
+
leftShoulder = recognizedPoints?.get(
503
+
VNHumanBodyPoseObservationJointNameLeftShoulder
504
+
)?.location?.toSkeletonPoint(width, height),
505
+
rightShoulder = recognizedPoints?.get(
506
+
VNHumanBodyPoseObservationJointNameRightShoulder
507
+
)?.location?.toSkeletonPoint(width, height),
508
+
leftElbow = recognizedPoints?.get(
509
+
VNHumanBodyPoseObservationJointNameLeftElbow
510
+
)?.location?.toSkeletonPoint(width, height),
511
+
rightElbow = recognizedPoints?.get(
512
+
VNHumanBodyPoseObservationJointNameRightElbow
513
+
)?.location?.toSkeletonPoint(width, height),
514
+
leftWrist = recognizedPoints?.get(
515
+
VNHumanBodyPoseObservationJointNameLeftWrist
516
+
)?.location?.toSkeletonPoint(width, height),
517
+
rightWrist = recognizedPoints?.get(
518
+
VNHumanBodyPoseObservationJointNameRightWrist
519
+
)?.location?.toSkeletonPoint(width, height),
520
+
leftHip = recognizedPoints?.get(
521
+
VNHumanBodyPoseObservationJointNameLeftHip
522
+
)?.location?.toSkeletonPoint(width, height),
523
+
rightHip = recognizedPoints?.get(
524
+
VNHumanBodyPoseObservationJointNameRightHip
525
+
)?.location?.toSkeletonPoint(width, height),
526
+
leftKnee = recognizedPoints?.get(
527
+
VNHumanBodyPoseObservationJointNameLeftKnee
528
+
)?.location?.toSkeletonPoint(width, height),
529
+
rightKnee = recognizedPoints?.get(
530
+
VNHumanBodyPoseObservationJointNameRightKnee
531
+
)?.location?.toSkeletonPoint(width, height),
532
+
leftAnkle = recognizedPoints?.get(
533
+
VNHumanBodyPoseObservationJointNameLeftAnkle
534
+
)?.location?.toSkeletonPoint(width, height),
535
+
rightAnkle = recognizedPoints?.get(
536
+
VNHumanBodyPoseObservationJointNameRightAnkle
537
+
)?.location?.toSkeletonPoint(width, height),
538
+
height = height.toFloat(),
539
+
width = width.toFloat()
540
+
)
541
+
onSkeletonProcessed(skelBuffer.smooth(updatedSkeleton))
542
+
}
543
+
}
544
+
}
545
+
requestForSkeleton.regionOfInterest = regionOfInterest
546
+
val handler = VNImageRequestHandler(cgImage, options)
547
+
handler.performRequests(
548
+
listOf(requestForObjects, requestForSkeleton), errorPtr.ptr
549
+
)
550
+
if (errorPtr.value != null) {
551
+
println("Error performing object detection request: ${errorPtr.value}")
552
+
onObjectsProcessed(emptyList())
553
+
onSkeletonProcessed(null)
554
+
}
555
+
} catch (e: Throwable) {
556
+
println("Unable to perform the object detection request: ${e.message}")
557
+
onObjectsProcessed(emptyList())
558
+
onSkeletonProcessed(null)
559
+
}
560
+
}
561
+
}
562
+
}
563
+
564
+
@OptIn(BetaInteropApi::class, ExperimentalNativeApi::class)
427
565
fun analyseBufferForAll(
428
566
buffer: CVImageBufferRef?,
429
567
timestamp: Long,
···
561
699
fun calculateRegionOfInterest(
562
700
recognizedPoints: MutableMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint>?
563
701
): CValue<CGRect> {
564
-
val margin = 0.1
702
+
val margin = 0.0
565
703
if (recognizedPoints.isNullOrEmpty()) {
566
704
return CGRectMake(0.0, 0.0, 1.0, 1.0) // Return full image area if no points are recognized
567
705
}
+8
-4
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.ios.kt
+8
-4
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.ios.kt
···
59
59
return suspendCancellableCoroutine { continuation ->
60
60
var poseResult: Skeleton? = null
61
61
var objectResults: List<AnalysisObject> = emptyList()
62
-
var completed = 0
63
-
62
+
/*var completed = 0
64
63
fun tryResume() {
65
64
completed++
66
65
if (completed == 2) {
67
66
continuation.resume(AnalysisResult(poseResult, objectResults))
68
67
}
69
68
}
70
-
71
69
frameProcessor.analyseFrame(img, inputFrame.timestamp, onProccessed = {
72
70
poseResult = it
73
71
tryResume()
74
72
})
75
-
76
73
frameProcessor.analyseFrameForObjects(img, inputFrame.timestamp, onProccessed = {
77
74
objectResults = it
78
75
tryResume()
76
+
})*/
77
+
78
+
frameProcessor.analyseFrameForAll(img, inputFrame.timestamp, onSkeletonProcessed = {
79
+
poseResult = it
80
+
}, onObjectsProcessed = {
81
+
objectResults = it
79
82
})
83
+
continuation.resume(AnalysisResult(poseResult, objectResults))
80
84
}
81
85
}
82
86
}