This repository has no description
refactor: consistency in buffer input
author
nate
date
(Feb 7, 2026, 12:49 PM +0200)
commit
6d819964
6d819964e12e6b4b30afbedeb02f2bdbcca62fa1
parent
10283ead
10283ead44df6b9abdef7013d66bc4142e5f020e
+83
-37
+1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
+1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
+82
-37
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
+82
-37
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
···
17
17
import kotlinx.cinterop.ptr
18
18
import kotlinx.cinterop.useContents
19
19
import kotlinx.cinterop.value
20
+
import platform.AVFoundation.AVCaptureConnection
21
+
import platform.AVFoundation.AVCaptureVideoOrientationLandscapeLeft
20
22
import platform.AVFoundation.AVCaptureVideoOrientationLandscapeRight
23
+
import platform.AVFoundation.AVCaptureVideoOrientationPortrait
24
+
import platform.AVFoundation.AVCaptureVideoOrientationPortraitUpsideDown
21
25
import platform.AVFoundation.AVCaptureVideoPreviewLayer
22
26
import platform.CoreFoundation.CFRelease
23
27
import platform.CoreFoundation.CFRetain
···
28
32
import platform.CoreGraphics.CGRect
29
33
import platform.CoreGraphics.CGRectMake
30
34
import platform.CoreVideo.CVImageBufferRef
35
+
import platform.CoreVideo.CVPixelBufferGetHeight
36
+
import platform.CoreVideo.CVPixelBufferGetWidth
31
37
import platform.Foundation.NSError
32
38
import platform.Foundation.NSUUID
39
+
import platform.ImageIO.kCGImagePropertyOrientationDown
40
+
import platform.ImageIO.kCGImagePropertyOrientationDownMirrored
41
+
import platform.ImageIO.kCGImagePropertyOrientationLeft
42
+
import platform.ImageIO.kCGImagePropertyOrientationLeftMirrored
43
+
import platform.ImageIO.kCGImagePropertyOrientationRight
44
+
import platform.ImageIO.kCGImagePropertyOrientationRightMirrored
45
+
import platform.ImageIO.kCGImagePropertyOrientationUp
46
+
import platform.ImageIO.kCGImagePropertyOrientationUpMirrored
33
47
import platform.Vision.VNClassificationObservation
34
48
import platform.Vision.VNCoreMLModel
35
49
import platform.Vision.VNCoreMLRequest
···
53
67
import platform.Vision.VNRecognizedObjectObservation
54
68
import platform.Vision.VNRecognizedPoint
55
69
import platform.Vision.VNRequest
56
-
import kotlin.experimental.ExperimentalNativeApi
57
-
import kotlin.math.absoluteValue
70
+
import kotlin.math.abs
58
71
import kotlin.math.max
59
72
import kotlin.math.min
60
-
import kotlin.native.identityHashCode
73
+
74
+
private const val VN_IMAGE_OPTION_CG_IMAGE_PROPERTY_ORIENTATION = "VNImageOptionCGImagePropertyOrientation"
61
75
62
76
fun bodyPoseHandler(request: VNRequest): List<MutableMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint>>? {
63
77
try {
···
134
148
}
135
149
136
150
@OptIn(ExperimentalForeignApi::class)
151
+
private fun AVCaptureConnection?.visionExifOrientation(): Int {
152
+
// Prefer the actual capture connection orientation (not UI orientation).
153
+
val videoOrientation = this?.videoOrientation ?: AVCaptureVideoOrientationLandscapeLeft
154
+
val mirrored = this?.videoMirrored == true
155
+
156
+
val exif: UInt = when (videoOrientation) {
157
+
// These mappings match Apple's docs: EXIF/CGImagePropertyOrientation values describe
158
+
// how to rotate the underlying pixel buffer into "Up".
159
+
AVCaptureVideoOrientationPortrait -> if (mirrored) kCGImagePropertyOrientationLeftMirrored else kCGImagePropertyOrientationRight
160
+
AVCaptureVideoOrientationPortraitUpsideDown -> if (mirrored) kCGImagePropertyOrientationRightMirrored else kCGImagePropertyOrientationLeft
161
+
AVCaptureVideoOrientationLandscapeLeft -> if (mirrored) kCGImagePropertyOrientationDownMirrored else kCGImagePropertyOrientationUp
162
+
AVCaptureVideoOrientationLandscapeRight -> if (mirrored) kCGImagePropertyOrientationUpMirrored else kCGImagePropertyOrientationDown
163
+
else -> if (mirrored) kCGImagePropertyOrientationDownMirrored else kCGImagePropertyOrientationUp
164
+
}
165
+
166
+
return exif.toInt()
167
+
}
168
+
169
+
@OptIn(ExperimentalForeignApi::class)
137
170
class FrameProcessor(var modelObj: VNCoreMLModel?) {
138
171
private val skelBuffer = SkelBuffer(maxSize = 10)
139
172
private var regionOfInterest = CGRectMake(0.0, 0.0, 1.0, 1.0)
···
182
215
}
183
216
}
184
217
185
-
@OptIn(BetaInteropApi::class, ExperimentalNativeApi::class)
218
+
@OptIn(BetaInteropApi::class)
186
219
fun analyseFrameForAll(
187
220
cgImage: CGImageRef?,
188
221
timestamp: Long,
···
239
272
}
240
273
}
241
274
AnalysisObject(
242
-
trackingId = observation.identityHashCode(),
275
+
trackingId = stableTrackingId(observation),
243
276
labels = labels,
244
277
boundingBox = boundingBox.normalize(),
245
278
frameSize = FrameSize(
246
-
width = width.toInt().absoluteValue,
247
-
height = height.toInt().absoluteValue
279
+
width = abs(width.toInt()),
280
+
height = abs(height.toInt())
248
281
)
249
282
)
250
283
}
251
284
onObjectsProcessed(analysisObjects)
285
+
}.apply {
286
+
// Make preprocessing consistent across orientations.
287
+
imageCropAndScaleOption = platform.Vision.VNImageCropAndScaleOptionCenterCrop
252
288
}
253
289
}
254
-
val options = mapOf<Any?, Any?>(
255
-
"orientation" to AVCaptureVideoOrientationLandscapeRight
290
+
// For CGImage path we don't have an AVCaptureConnection; assume the CGImage
291
+
// is already "up".
292
+
val options: Map<Any?, Any?> = mapOf(
293
+
VN_IMAGE_OPTION_CG_IMAGE_PROPERTY_ORIENTATION to kCGImagePropertyOrientationUp.toInt()
256
294
)
295
+
257
296
val requestForSkeleton = VNDetectHumanBodyPoseRequest { request, error ->
258
297
if (error != null) {
259
298
onSkeletonProcessed(null)
···
363
402
}
364
403
}
365
404
366
-
@OptIn(BetaInteropApi::class, ExperimentalNativeApi::class)
405
+
@OptIn(BetaInteropApi::class)
367
406
fun analyseBufferForAll(
368
407
buffer: CVImageBufferRef?,
369
408
timestamp: Long,
370
409
preview: AVCaptureVideoPreviewLayer?,
410
+
captureConnection: AVCaptureConnection? = preview?.connection,
371
411
onObjectsProcessed: (List<AnalysisObject>) -> Unit,
372
412
onSkeletonProcessed: (Skeleton?) -> Unit
373
413
) {
···
378
418
return
379
419
}
380
420
val retainedBuffer = CFRetain(buffer)
381
-
val width = 480uL // You may want to get actual width from buffer if needed
382
-
val height = 360uL // You may want to get actual height from buffer if needed
421
+
val width = CVPixelBufferGetWidth(buffer).toULong()
422
+
val height = CVPixelBufferGetHeight(buffer).toULong()
383
423
memScoped {
384
424
val errorPtr = alloc<ObjCObjectVar<NSError?>>()
385
425
try {
386
-
val requestForObjects = if(detectMode.doObject()){
426
+
val requestForObjects = if (detectMode.doObject()) {
387
427
modelObj?.let {
388
428
VNCoreMLRequest(it) { request, error ->
389
429
if (error != null) {
···
418
458
}
419
459
}
420
460
AnalysisObject(
421
-
trackingId = observation.identityHashCode(),
461
+
trackingId = stableTrackingId(observation),
422
462
labels = labels,
423
463
boundingBox = boundingBox.normalize(),
424
464
frameSize = FrameSize(
425
-
width = width.toInt().absoluteValue,
426
-
height = height.toInt().absoluteValue
465
+
width = abs(width.toInt()),
466
+
height = abs(height.toInt())
427
467
)
428
468
)
429
469
}
430
470
onObjectsProcessed(analysisObjects)
471
+
}.apply {
472
+
imageCropAndScaleOption = platform.Vision.VNImageCropAndScaleOptionCenterCrop
431
473
}
432
474
}
433
-
}else null
434
-
475
+
} else null
435
476
436
-
val options = mapOf<Any?, Any?>(
437
-
"orientation" to AVCaptureVideoOrientationLandscapeRight
477
+
val options: Map<Any?, Any?> = mapOf(
478
+
VN_IMAGE_OPTION_CG_IMAGE_PROPERTY_ORIENTATION to captureConnection.visionExifOrientation()
438
479
)
439
-
val requestForSkeleton = if(detectMode.doPose()) {
480
+
481
+
val requestForSkeleton = if (detectMode.doPose()) {
440
482
VNDetectHumanBodyPoseRequest { request, error ->
441
483
if (error != null) {
442
484
onSkeletonProcessed(null)
443
485
} else {
444
486
request?.also { vnRequest ->
445
-
val recognizedPoints = bodyPoseHandler(vnRequest)?.firstOrNull {
487
+
val recognizedPoints = bodyPoseHandler(vnRequest)?.firstOrNull { pointsMap ->
446
488
Skeleton(
447
489
timestamp = timestamp,
448
-
leftShoulder = it[VNHumanBodyPoseObservationJointNameLeftShoulder]?.location?.toSkeletonPoint(
490
+
leftShoulder = pointsMap[VNHumanBodyPoseObservationJointNameLeftShoulder]?.location?.toSkeletonPoint(
449
491
width,
450
492
height
451
493
),
452
-
rightShoulder = it[VNHumanBodyPoseObservationJointNameRightShoulder]?.location?.toSkeletonPoint(
494
+
rightShoulder = pointsMap[VNHumanBodyPoseObservationJointNameRightShoulder]?.location?.toSkeletonPoint(
453
495
width,
454
496
height
455
497
),
456
-
leftElbow = it[VNHumanBodyPoseObservationJointNameLeftElbow]?.location?.toSkeletonPoint(
498
+
leftElbow = pointsMap[VNHumanBodyPoseObservationJointNameLeftElbow]?.location?.toSkeletonPoint(
457
499
width,
458
500
height
459
501
),
460
-
rightElbow = it[VNHumanBodyPoseObservationJointNameRightElbow]?.location?.toSkeletonPoint(
502
+
rightElbow = pointsMap[VNHumanBodyPoseObservationJointNameRightElbow]?.location?.toSkeletonPoint(
461
503
width,
462
504
height
463
505
),
464
-
leftWrist = it[VNHumanBodyPoseObservationJointNameLeftWrist]?.location?.toSkeletonPoint(
506
+
leftWrist = pointsMap[VNHumanBodyPoseObservationJointNameLeftWrist]?.location?.toSkeletonPoint(
465
507
width,
466
508
height
467
509
),
468
-
rightWrist = it[VNHumanBodyPoseObservationJointNameRightWrist]?.location?.toSkeletonPoint(
510
+
rightWrist = pointsMap[VNHumanBodyPoseObservationJointNameRightWrist]?.location?.toSkeletonPoint(
469
511
width,
470
512
height
471
513
),
472
-
leftHip = it[VNHumanBodyPoseObservationJointNameLeftHip]?.location?.toSkeletonPoint(
514
+
leftHip = pointsMap[VNHumanBodyPoseObservationJointNameLeftHip]?.location?.toSkeletonPoint(
473
515
width,
474
516
height
475
517
),
476
-
rightHip = it[VNHumanBodyPoseObservationJointNameRightHip]?.location?.toSkeletonPoint(
518
+
rightHip = pointsMap[VNHumanBodyPoseObservationJointNameRightHip]?.location?.toSkeletonPoint(
477
519
width,
478
520
height
479
521
),
480
-
leftKnee = it[VNHumanBodyPoseObservationJointNameLeftKnee]?.location?.toSkeletonPoint(
522
+
leftKnee = pointsMap[VNHumanBodyPoseObservationJointNameLeftKnee]?.location?.toSkeletonPoint(
481
523
width,
482
524
height
483
525
),
484
-
rightKnee = it[VNHumanBodyPoseObservationJointNameRightKnee]?.location?.toSkeletonPoint(
526
+
rightKnee = pointsMap[VNHumanBodyPoseObservationJointNameRightKnee]?.location?.toSkeletonPoint(
485
527
width,
486
528
height
487
529
),
488
-
leftAnkle = it[VNHumanBodyPoseObservationJointNameLeftAnkle]?.location?.toSkeletonPoint(
530
+
leftAnkle = pointsMap[VNHumanBodyPoseObservationJointNameLeftAnkle]?.location?.toSkeletonPoint(
489
531
width,
490
532
height
491
533
),
492
-
rightAnkle = it[VNHumanBodyPoseObservationJointNameRightAnkle]?.location?.toSkeletonPoint(
534
+
rightAnkle = pointsMap[VNHumanBodyPoseObservationJointNameRightAnkle]?.location?.toSkeletonPoint(
493
535
width,
494
536
height
495
537
),
···
506
548
}
507
549
}?.isInFocusArea(focusArea) ?: false
508
550
}
509
-
regionOfInterest =
510
-
calculateRegionOfInterest(recognizedPoints)
551
+
552
+
regionOfInterest = calculateRegionOfInterest(recognizedPoints)
553
+
511
554
val updatedSkeleton = Skeleton(
512
555
timestamp = timestamp,
513
556
leftShoulder = recognizedPoints?.get(
···
549
592
height = height.toFloat(),
550
593
width = width.toFloat()
551
594
)
595
+
552
596
onSkeletonProcessed(skelBuffer.smooth(updatedSkeleton))
553
597
}
554
598
}
···
636
680
}
637
681
}
638
682
683
+
private fun stableTrackingId(observation: VNRecognizedObjectObservation): Int = observation.hashCode()
684
+
639
685
data class DetectedObject @OptIn(ExperimentalForeignApi::class) constructor(
640
686
val id: String = NSUUID().UUIDString(),
641
687
val label: String,
642
688
val confidence: Float,
643
689
val boundingBox: CValue<CGRect>
644
690
)
645
-