This repository has no description
Merge branch 'more-performant-ios'
# Conflicts:
# posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
# posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/skeleton/Skeleton.kt
# posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
# posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt
# posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
author
nathan holland
date
(May 22, 2025, 3:17 PM +0300)
commit
a2739c10
a2739c10fd79a7cbcfc2172e73ffe781717d8657
parent
e68f31fa
e68f31fa412a920600a5559bb565abf61dd8077f
+84
-71
+52
-48
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
+52
-48
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
···
86
86
import kotlin.native.runtime.NativeRuntimeApi
87
87
88
88
class CameraEngine : UIViewController(null, null) {
89
-
private var isCapturing = atomic(false)
90
89
val cameraController = CameraController()
91
90
92
91
private val memoryManager = MemoryManager
···
373
372
}
374
373
375
374
private val isProcessing = atomic(false)
376
-
private val scope = CoroutineScope(Job() + Dispatchers.Main)
377
-
378
-
@OptIn(ExperimentalForeignApi::class)
379
-
private var frameProcessor = FrameProcessor()
380
375
381
376
@OptIn(ExperimentalForeignApi::class, NativeRuntimeApi::class)
382
377
override fun captureOutput(
···
384
379
didOutputSampleBuffer: CMSampleBufferRef?,
385
380
fromConnection: AVCaptureConnection
386
381
) {
387
-
val timestamp = timeStamp()
388
-
if (isProcessing.compareAndSet(expect = false, update = true)) {
389
-
try {
390
-
kotlin.native.runtime.GC.collect()
391
-
val result = runCatching {
392
-
393
-
val buffer = CMSampleBufferGetImageBuffer(didOutputSampleBuffer)
394
-
val w = CVPixelBufferGetWidth(buffer)
395
-
val h = CVPixelBufferGetHeight(buffer)
396
-
val imageBuffer = CIImage.imageWithCVPixelBuffer(buffer)
397
-
val cgImage = imageBuffer.toCGImageRef(w, h)
398
-
val processingQueue = dispatch_queue_create("com.performancecoachlab.frameProcessing", null)
399
-
dispatch_async(processingQueue) {
400
-
frameProcessor.processPose(cgImage, timestamp) { skeleton ->
401
-
cameraPreviewLayer?.also { preview->
402
-
403
-
val updatedSkeleton = skeleton?.let {
404
-
mapSkeletonToPreview(
405
-
skeleton = it,
406
-
previewLayer = preview,
407
-
isUsingFrontCamera = isUsingFrontCamera,
408
-
width = w.toFloat(),
409
-
height = h.toFloat()
410
-
)
382
+
MemoryManager.updateMemoryStatus()
383
+
kotlin.native.runtime.GC.collect()
384
+
timeStamp().also { timestamp->
385
+
runCatching {
386
+
CMSampleBufferGetImageBuffer(didOutputSampleBuffer).also { buffer ->
387
+
CVPixelBufferGetWidth(buffer).also { w ->
388
+
CVPixelBufferGetHeight(buffer).also { h ->
389
+
CIImage.imageWithCVPixelBuffer(buffer).also { imageBuffer ->
390
+
imageBuffer.toCGImageRef(w, h).also { cgImage ->
391
+
dispatch_queue_create(
392
+
"com.performancecoachlab.frameProcessing", null
393
+
).also { processingQueue ->
394
+
dispatch_async(processingQueue) {
395
+
processPose(
396
+
cgImage, timestamp
397
+
) { skeleton ->
398
+
cameraPreviewLayer?.also { preview ->
399
+
skeleton?.let {
400
+
mapSkeletonToPreview(
401
+
skeleton = it,
402
+
previewLayer = preview,
403
+
isUsingFrontCamera = isUsingFrontCamera,
404
+
width = w.toFloat(),
405
+
height = h.toFloat()
406
+
)
407
+
}?.also { updatedSkeleton ->
408
+
skeletonRepository?.updateSkeleton(
409
+
updatedSkeleton
410
+
)
411
+
preview.bounds.useContents {
412
+
Pair(
413
+
size.width.toInt(),
414
+
size.height.toInt()
415
+
)
416
+
}.also { bo ->
417
+
ImageBitmap(
418
+
bo.first, bo.second
419
+
).drawSkeleton(inskeleton = updatedSkeleton)
420
+
.also { drawn ->
421
+
frameListener?.updateFrame(
422
+
drawn
423
+
)
424
+
}
425
+
}
426
+
}
427
+
}
428
+
CGImageRelease(cgImage)
429
+
}
430
+
}
431
+
}
432
+
}
411
433
}
412
-
updatedSkeleton?.also { skel ->
413
-
skeletonRepository?.updateSkeleton(skel)
414
-
}
415
-
val bo = preview.bounds.useContents { Pair(size.width.toInt(), size.height.toInt()) }
416
-
val drawn = ImageBitmap(bo.first,bo.second).drawSkeleton(inskeleton = updatedSkeleton)
417
-
frameListener?.updateFrame(drawn)
418
434
}
419
-
CGImageRelease(cgImage)
420
435
}
421
436
}
422
-
}
423
-
424
-
if (result.isFailure) {
425
-
println("Exception during performRequests: ${result.exceptionOrNull()?.message}")
426
-
}
427
-
} catch (e: Exception) {
428
-
println("Error during frame processing: ${e.message}")
429
-
} catch (e: Throwable){
430
-
println("Error during frame processing: ${e.message}")
431
-
}finally {
432
-
isProcessing.value = false
433
-
//kotlin.native.runtime.GC.collect()
434
437
}
438
+
435
439
}
436
440
}
437
441
+3
-2
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt
+3
-2
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt
···
12
12
import androidx.compose.ui.Modifier
13
13
import androidx.compose.ui.layout.ContentScale
14
14
import com.performancecoachlab.posedetection.skeleton.SkeletonRepository
15
+
import kotlinx.cinterop.BetaInteropApi
16
+
import kotlinx.cinterop.autoreleasepool
15
17
import kotlin.native.runtime.NativeRuntimeApi
16
18
17
-
@OptIn(NativeRuntimeApi::class)
19
+
@OptIn(NativeRuntimeApi::class, BetaInteropApi::class)
18
20
@Composable
19
21
actual fun CameraView(
20
22
skeletonRepository: SkeletonRepository,
···
46
48
modifier = Modifier.fillMaxSize(),
47
49
contentScale = ContentScale.Crop,
48
50
)
49
-
//kotlin.native.runtime.GC.collect()
50
51
}
51
52
}
52
53
+26
-20
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
+26
-20
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
···
6
6
import kotlinx.cinterop.ExperimentalForeignApi
7
7
import kotlinx.cinterop.ObjCObjectVar
8
8
import kotlinx.cinterop.alloc
9
+
import kotlinx.cinterop.autoreleasepool
9
10
import kotlinx.cinterop.memScoped
10
11
import kotlinx.cinterop.ptr
11
12
import kotlinx.cinterop.useContents
···
48
49
}
49
50
}
50
51
51
-
@OptIn(ExperimentalForeignApi::class)
52
-
fun processObservation(observation: VNHumanBodyPoseObservation): MutableMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint> {
53
-
val points = emptyMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint>().toMutableMap()
54
-
observation.availableJointNames.forEach {
55
-
observation.recognizedPointForJointName(it as VNHumanBodyPoseObservationJointName, null)
56
-
?.also { point ->
57
-
if (point.confidence > 0f) {
58
-
points[it] = point
59
-
}
52
+
@OptIn(ExperimentalForeignApi::class)
53
+
fun processObservation(observation: VNHumanBodyPoseObservation): MutableMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint> {
54
+
val points = emptyMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint>().toMutableMap()
55
+
observation.availableJointNames.forEach {
56
+
observation.recognizedPointForJointName(it as VNHumanBodyPoseObservationJointName, null)
57
+
?.also { point ->
58
+
if (point.confidence > 0f) {
59
+
points[it] = point
60
60
}
61
-
}
62
-
return points
61
+
}
63
62
}
63
+
return points
64
+
}
64
65
65
-
@OptIn(ExperimentalForeignApi::class)
66
-
fun CValue<CGPoint>.toSkeletonPoint(width: ULong, height: ULong): Skeleton.SkeletonCoordinate {
67
-
return VNImagePointForNormalizedPoint(
68
-
this, width, height
69
-
).let {
70
-
it.useContents { Skeleton.SkeletonCoordinate(x.toFloat(), height.toFloat() - y.toFloat()) }
66
+
@OptIn(ExperimentalForeignApi::class)
67
+
fun CValue<CGPoint>.toSkeletonPoint(width: ULong, height: ULong): Skeleton.SkeletonCoordinate {
68
+
return VNImagePointForNormalizedPoint(
69
+
this, width, height
70
+
).let {
71
+
it.useContents {
72
+
Skeleton.SkeletonCoordinate(
73
+
x.toFloat(), height.toFloat() - y.toFloat()
74
+
)
71
75
}
72
76
}
77
+
}
73
78
74
-
@OptIn(ExperimentalForeignApi::class, NativeRuntimeApi::class, BetaInteropApi::class)
75
-
fun processPose(cgImage: CGImageRef?, timestamp: Long, onProccessed: (Skeleton?) -> Unit) {
79
+
@OptIn(ExperimentalForeignApi::class, NativeRuntimeApi::class, BetaInteropApi::class)
80
+
fun processPose(cgImage: CGImageRef?, timestamp: Long, onProccessed: (Skeleton?) -> Unit) {
81
+
autoreleasepool {
76
82
if (cgImage == null) {
77
83
onProccessed(null)
78
84
return
79
85
}
80
86
val width = CGImageGetWidth(cgImage)
81
87
val height = CGImageGetHeight(cgImage)
82
-
if(width.toUInt() == 0u || height.toUInt() == 0u){
88
+
if (width.toUInt() == 0u || height.toUInt() == 0u) {
83
89
onProccessed(null)
84
90
return
85
91
}
+3
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/MemoryManager.kt
+3
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/MemoryManager.kt
···
6
6
import platform.Foundation.NSOperationQueue
7
7
import platform.Foundation.NSProcessInfo
8
8
import platform.UIKit.UIApplicationDidReceiveMemoryWarningNotification
9
+
import kotlin.native.runtime.NativeRuntimeApi
9
10
10
11
/**
11
12
* Manages memory resources for camera operations
···
13
14
*/
14
15
object MemoryManager {
15
16
16
-
private const val MEMORY_PRESSURE_THRESHOLD = 0.8
17
+
private const val MEMORY_PRESSURE_THRESHOLD = 0.4
17
18
18
19
19
20
private val smallBufferLock = NSLock()
···
79
80
/**
80
81
* Handle high memory pressure situation
81
82
*/
83
+
@OptIn(NativeRuntimeApi::class)
82
84
private fun handleHighMemoryPressure() {
83
85
clearBufferPools()
84
86
}