This repository has no description
refactor: remove unnecessary code from earlier iterations
author
nathan holland
date
(May 5, 2025, 4:05 PM +0300)
commit
e3b6d1d0
e3b6d1d0f3d6d57359d735403760e501f3ad6346
parent
3563637a
3563637abcbc646c2a9eb2d77b19ff28e3aef687
+223
-166
+115
-50
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
+115
-50
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
···
17
17
import kotlinx.cinterop.refTo
18
18
import kotlinx.cinterop.useContents
19
19
import kotlinx.cinterop.usePinned
20
-
import kotlinx.coroutines.CoroutineScope
21
-
import kotlinx.coroutines.Dispatchers
22
-
import kotlinx.coroutines.Job
23
20
import org.jetbrains.skia.ColorAlphaType
24
21
import org.jetbrains.skia.ColorType
25
22
import org.jetbrains.skia.Image
···
86
83
import kotlin.native.runtime.NativeRuntimeApi
87
84
88
85
class CameraEngine : UIViewController(null, null) {
89
-
private var isCapturing = atomic(false)
90
86
val cameraController = CameraController()
91
87
92
88
private val memoryManager = MemoryManager
···
373
369
}
374
370
375
371
private val isProcessing = atomic(false)
376
-
private val scope = CoroutineScope(Job() + Dispatchers.Main)
377
-
378
-
@OptIn(ExperimentalForeignApi::class)
379
-
private var frameProcessor = FrameProcessor()
380
372
381
373
@OptIn(ExperimentalForeignApi::class, NativeRuntimeApi::class)
382
374
override fun captureOutput(
···
384
376
didOutputSampleBuffer: CMSampleBufferRef?,
385
377
fromConnection: AVCaptureConnection
386
378
) {
387
-
val timestamp = timeStamp()
388
-
if (isProcessing.compareAndSet(expect = false, update = true)) {
389
-
try {
390
-
kotlin.native.runtime.GC.collect()
391
-
val result = runCatching {
392
-
393
-
val buffer = CMSampleBufferGetImageBuffer(didOutputSampleBuffer)
394
-
val w = CVPixelBufferGetWidth(buffer)
395
-
val h = CVPixelBufferGetHeight(buffer)
396
-
val imageBuffer = CIImage.imageWithCVPixelBuffer(buffer)
397
-
val cgImage = imageBuffer.toCGImageRef(w, h)
398
-
val processingQueue = dispatch_queue_create("com.performancecoachlab.frameProcessing", null)
399
-
dispatch_async(processingQueue) {
400
-
frameProcessor.processPose(cgImage, timestamp) { skeleton ->
401
-
cameraPreviewLayer?.also { preview->
402
-
403
-
val updatedSkeleton = skeleton?.let {
404
-
mapSkeletonToPreview(
405
-
skeleton = it,
406
-
previewLayer = preview,
407
-
isUsingFrontCamera = isUsingFrontCamera,
408
-
width = w.toFloat(),
409
-
height = h.toFloat()
410
-
)
379
+
MemoryManager.updateMemoryStatus()
380
+
kotlin.native.runtime.GC.collect()
381
+
timeStamp().also { timestamp->
382
+
runCatching {
383
+
CMSampleBufferGetImageBuffer(didOutputSampleBuffer).also { buffer ->
384
+
CVPixelBufferGetWidth(buffer).also { w ->
385
+
CVPixelBufferGetHeight(buffer).also { h ->
386
+
CIImage.imageWithCVPixelBuffer(buffer).also { imageBuffer ->
387
+
imageBuffer.toCGImageRef(w, h).also { cgImage ->
388
+
dispatch_queue_create(
389
+
"com.performancecoachlab.frameProcessing", null
390
+
).also { processingQueue ->
391
+
dispatch_async(processingQueue) {
392
+
processPose(
393
+
cgImage, timestamp
394
+
) { skeleton ->
395
+
cameraPreviewLayer?.also { preview ->
396
+
skeleton?.let {
397
+
mapSkeletonToPreview(
398
+
skeleton = it,
399
+
previewLayer = preview,
400
+
isUsingFrontCamera = isUsingFrontCamera,
401
+
width = w.toFloat(),
402
+
height = h.toFloat()
403
+
)
404
+
}?.also { updatedSkeleton ->
405
+
skeletonRepository?.updateSkeleton(
406
+
updatedSkeleton
407
+
)
408
+
preview.bounds.useContents {
409
+
Pair(
410
+
size.width.toInt(),
411
+
size.height.toInt()
412
+
)
413
+
}.also { bo ->
414
+
ImageBitmap(
415
+
bo.first, bo.second
416
+
).drawSkeleton(inskeleton = updatedSkeleton)
417
+
.also { drawn ->
418
+
frameListener?.updateFrame(
419
+
drawn
420
+
)
421
+
}
422
+
}
423
+
}
424
+
}
425
+
CGImageRelease(cgImage)
426
+
}
427
+
}
428
+
}
429
+
}
411
430
}
412
-
updatedSkeleton?.also { skel ->
413
-
skeletonRepository?.updateSkeleton(skel)
414
-
}
415
-
val bo = preview.bounds.useContents { Pair(size.width.toInt(), size.height.toInt()) }
416
-
val drawn = ImageBitmap(bo.first,bo.second).drawSkeleton(inskeleton = updatedSkeleton)
417
-
frameListener?.updateFrame(drawn)
418
431
}
419
-
CGImageRelease(cgImage)
420
432
}
421
433
}
422
-
}
434
+
}
423
435
424
-
if (result.isFailure) {
425
-
println("Exception during performRequests: ${result.exceptionOrNull()?.message}")
436
+
}
437
+
/*if (isProcessing.compareAndSet(expect = false, update = true)) {
438
+
try {
439
+
runCatching {
440
+
CMSampleBufferGetImageBuffer(didOutputSampleBuffer).also { buffer ->
441
+
CVPixelBufferGetWidth(buffer).also { w ->
442
+
CVPixelBufferGetHeight(buffer).also { h ->
443
+
CIImage.imageWithCVPixelBuffer(buffer).also { imageBuffer ->
444
+
imageBuffer.toCGImageRef(w, h).also { cgImage ->
445
+
dispatch_queue_create(
446
+
"com.performancecoachlab.frameProcessing", null
447
+
).also { processingQueue ->
448
+
dispatch_async(processingQueue) {
449
+
processPose(
450
+
cgImage, timestamp
451
+
) { skeleton ->
452
+
cameraPreviewLayer?.also { preview ->
453
+
skeleton?.let {
454
+
mapSkeletonToPreview(
455
+
skeleton = it,
456
+
previewLayer = preview,
457
+
isUsingFrontCamera = isUsingFrontCamera,
458
+
width = w.toFloat(),
459
+
height = h.toFloat()
460
+
)
461
+
}?.also { updatedSkeleton ->
462
+
skeletonRepository?.updateSkeleton(
463
+
updatedSkeleton
464
+
)
465
+
preview.bounds.useContents {
466
+
Pair(
467
+
size.width.toInt(),
468
+
size.height.toInt()
469
+
)
470
+
}.also { bo ->
471
+
ImageBitmap(
472
+
bo.first, bo.second
473
+
).drawSkeleton(inskeleton = updatedSkeleton)
474
+
.also { drawn ->
475
+
frameListener?.updateFrame(
476
+
drawn
477
+
)
478
+
}
479
+
}
480
+
}
481
+
}
482
+
CGImageRelease(cgImage)
483
+
}
484
+
}
485
+
}
486
+
}
487
+
}
488
+
}
489
+
}
490
+
}
426
491
}
427
-
} catch (e: Exception) {
492
+
} catch (e: Throwable) {
428
493
println("Error during frame processing: ${e.message}")
429
-
} catch (e: Throwable){
430
-
println("Error during frame processing: ${e.message}")
431
-
}finally {
494
+
} finally {
432
495
isProcessing.value = false
433
-
//kotlin.native.runtime.GC.collect()
434
496
}
435
-
}
497
+
}*/
436
498
}
437
499
438
500
private inline fun guard(condition: Boolean, crossinline block: () -> Unit) {
···
461
523
if (point == null) return null
462
524
463
525
// Normalize the point
464
-
val normalizedPoint = CGPointMake(point.x.toDouble()/width, point.y.toDouble()/height)
526
+
val normalizedPoint =
527
+
CGPointMake(point.x.toDouble() / width, point.y.toDouble() / height)
465
528
val screenPoint = previewLayer.pointForCaptureDevicePointOfInterest(normalizedPoint)
466
-
return Skeleton.SkeletonCoordinate(screenPoint.useContents { x.toFloat() }, screenPoint.useContents { y.toFloat() })
529
+
return Skeleton.SkeletonCoordinate(
530
+
screenPoint.useContents { x.toFloat() },
531
+
screenPoint.useContents { y.toFloat() })
467
532
}
468
533
469
534
return Skeleton(
+13
-10
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt
+13
-10
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt
···
12
12
import androidx.compose.ui.Modifier
13
13
import androidx.compose.ui.layout.ContentScale
14
14
import com.performancecoachlab.posedetection.skeleton.SkeletonRepository
15
+
import kotlinx.cinterop.BetaInteropApi
16
+
import kotlinx.cinterop.autoreleasepool
15
17
import kotlin.native.runtime.NativeRuntimeApi
16
18
17
-
@OptIn(NativeRuntimeApi::class)
19
+
@OptIn(NativeRuntimeApi::class, BetaInteropApi::class)
18
20
@Composable
19
21
actual fun CameraView(
20
22
skeletonRepository: SkeletonRepository,
···
38
40
cameraEngine.value = engine.also { if (!frontCamera) it.toggleCameraLens() }
39
41
})
40
42
if (drawSkeleton) {
41
-
kotlin.native.runtime.GC.collect()
42
-
frameBitmap?.also {
43
-
Image(
44
-
bitmap = it,
45
-
contentDescription = "video frame",
46
-
modifier = Modifier.fillMaxSize(),
47
-
contentScale = ContentScale.Crop,
48
-
)
49
-
//kotlin.native.runtime.GC.collect()
43
+
autoreleasepool {
44
+
frameBitmap?.also {
45
+
Image(
46
+
bitmap = it,
47
+
contentDescription = "video frame",
48
+
modifier = Modifier.fillMaxSize(),
49
+
contentScale = ContentScale.Crop,
50
+
)
51
+
}
50
52
}
53
+
51
54
}
52
55
53
56
}
+92
-105
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
+92
-105
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
···
6
6
import kotlinx.cinterop.ExperimentalForeignApi
7
7
import kotlinx.cinterop.ObjCObjectVar
8
8
import kotlinx.cinterop.alloc
9
+
import kotlinx.cinterop.autoreleasepool
9
10
import kotlinx.cinterop.memScoped
10
11
import kotlinx.cinterop.ptr
11
12
import kotlinx.cinterop.useContents
12
-
import kotlinx.cinterop.value
13
13
import platform.CoreGraphics.CGImageGetHeight
14
14
import platform.CoreGraphics.CGImageGetWidth
15
15
import platform.CoreGraphics.CGImageRef
···
36
36
import platform.Vision.VNRequest
37
37
import kotlin.native.runtime.NativeRuntimeApi
38
38
39
-
class FrameProcessor {
40
-
fun bodyPoseHandler(request: VNRequest): MutableMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint>? {
41
-
try {
42
-
val observations = request.results as List<VNHumanBodyPoseObservation>
43
-
// Process each observation to find the recognized body pose points.
44
-
return observations.lastOrNull()?.let { processObservation(it) }
45
-
} catch (e: Exception) {
46
-
println("Error processing observations: ${e.message}")
47
-
return null
48
-
}
39
+
fun bodyPoseHandler(request: VNRequest): MutableMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint>? {
40
+
try {
41
+
val observations = request.results as List<VNHumanBodyPoseObservation>
42
+
// Process each observation to find the recognized body pose points.
43
+
return observations.lastOrNull()?.let { processObservation(it) }
44
+
} catch (e: Exception) {
45
+
println("Error processing observations: ${e.message}")
46
+
return null
49
47
}
48
+
}
50
49
51
-
@OptIn(ExperimentalForeignApi::class)
52
-
fun processObservation(observation: VNHumanBodyPoseObservation): MutableMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint> {
53
-
val points = emptyMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint>().toMutableMap()
54
-
observation.availableJointNames.forEach {
55
-
observation.recognizedPointForJointName(it as VNHumanBodyPoseObservationJointName, null)
56
-
?.also { point ->
57
-
if (point.confidence > 0f) {
58
-
points[it] = point
59
-
}
50
+
@OptIn(ExperimentalForeignApi::class)
51
+
fun processObservation(observation: VNHumanBodyPoseObservation): MutableMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint> {
52
+
val points = emptyMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint>().toMutableMap()
53
+
observation.availableJointNames.forEach {
54
+
observation.recognizedPointForJointName(it as VNHumanBodyPoseObservationJointName, null)
55
+
?.also { point ->
56
+
if (point.confidence > 0f) {
57
+
points[it] = point
60
58
}
61
-
}
62
-
return points
59
+
}
63
60
}
61
+
return points
62
+
}
64
63
65
-
@OptIn(ExperimentalForeignApi::class)
66
-
fun CValue<CGPoint>.toSkeletonPoint(width: ULong, height: ULong): Skeleton.SkeletonCoordinate {
67
-
return VNImagePointForNormalizedPoint(
68
-
this, width, height
69
-
).let {
70
-
it.useContents { Skeleton.SkeletonCoordinate(x.toFloat(), height.toFloat() - y.toFloat()) }
64
+
@OptIn(ExperimentalForeignApi::class)
65
+
fun CValue<CGPoint>.toSkeletonPoint(width: ULong, height: ULong): Skeleton.SkeletonCoordinate {
66
+
return VNImagePointForNormalizedPoint(
67
+
this, width, height
68
+
).let {
69
+
it.useContents {
70
+
Skeleton.SkeletonCoordinate(
71
+
x.toFloat(), height.toFloat() - y.toFloat()
72
+
)
71
73
}
72
74
}
75
+
}
73
76
74
-
@OptIn(ExperimentalForeignApi::class, NativeRuntimeApi::class, BetaInteropApi::class)
75
-
fun processPose(cgImage: CGImageRef?, timestamp: Long, onProccessed: (Skeleton?) -> Unit) {
77
+
@OptIn(ExperimentalForeignApi::class, NativeRuntimeApi::class, BetaInteropApi::class)
78
+
fun processPose(cgImage: CGImageRef?, timestamp: Long, onProccessed: (Skeleton?) -> Unit) {
79
+
autoreleasepool {
76
80
if (cgImage == null) {
77
81
onProccessed(null)
78
82
return
79
83
}
80
84
val width = CGImageGetWidth(cgImage)
81
85
val height = CGImageGetHeight(cgImage)
82
-
if(width.toUInt() == 0u || height.toUInt() == 0u){
86
+
if (width.toUInt() == 0u || height.toUInt() == 0u) {
83
87
onProccessed(null)
84
88
return
85
89
}
86
90
memScoped {
87
-
val errorPtr = alloc<ObjCObjectVar<NSError?>>()
88
-
val requestHandler = VNImageRequestHandler(cgImage, mapOf<Any?, Any?>())
89
-
val request = VNDetectHumanBodyPoseRequest { request, error ->
90
-
if (error != null) {
91
-
println("Unable to perform the request: $error")
92
-
onProccessed(null)
93
-
} else {
94
-
request?.also { vnRequest ->
95
-
val recognizedPoints = bodyPoseHandler(vnRequest)
96
-
val updatedSkeleton = Skeleton(
97
-
timestamp = timestamp,
98
-
leftShoulder = recognizedPoints?.get(
99
-
VNHumanBodyPoseObservationJointNameLeftShoulder
100
-
)?.location?.toSkeletonPoint(width, height),
101
-
rightShoulder = recognizedPoints?.get(
102
-
VNHumanBodyPoseObservationJointNameRightShoulder
103
-
)?.location?.toSkeletonPoint(width, height),
104
-
leftElbow = recognizedPoints?.get(
105
-
VNHumanBodyPoseObservationJointNameLeftElbow
106
-
)?.location?.toSkeletonPoint(width, height),
107
-
rightElbow = recognizedPoints?.get(
108
-
VNHumanBodyPoseObservationJointNameRightElbow
109
-
)?.location?.toSkeletonPoint(width, height),
110
-
leftWrist = recognizedPoints?.get(
111
-
VNHumanBodyPoseObservationJointNameLeftWrist
112
-
)?.location?.toSkeletonPoint(width, height),
113
-
rightWrist = recognizedPoints?.get(
114
-
VNHumanBodyPoseObservationJointNameRightWrist
115
-
)?.location?.toSkeletonPoint(width, height),
116
-
leftHip = recognizedPoints?.get(
117
-
VNHumanBodyPoseObservationJointNameLeftHip
118
-
)?.location?.toSkeletonPoint(width, height),
119
-
rightHip = recognizedPoints?.get(
120
-
VNHumanBodyPoseObservationJointNameRightHip
121
-
)?.location?.toSkeletonPoint(width, height),
122
-
leftKnee = recognizedPoints?.get(
123
-
VNHumanBodyPoseObservationJointNameLeftKnee
124
-
)?.location?.toSkeletonPoint(width, height),
125
-
rightKnee = recognizedPoints?.get(
126
-
VNHumanBodyPoseObservationJointNameRightKnee
127
-
)?.location?.toSkeletonPoint(width, height),
128
-
leftAnkle = recognizedPoints?.get(
129
-
VNHumanBodyPoseObservationJointNameLeftAnkle
130
-
)?.location?.toSkeletonPoint(width, height),
131
-
rightAnkle = recognizedPoints?.get(
132
-
VNHumanBodyPoseObservationJointNameRightAnkle
133
-
)?.location?.toSkeletonPoint(width, height)
134
-
)
135
-
onProccessed(updatedSkeleton)
91
+
alloc<ObjCObjectVar<NSError?>>().also { errorPtr ->
92
+
VNImageRequestHandler(cgImage, mapOf<Any?, Any?>()).also { requestHandler ->
93
+
VNDetectHumanBodyPoseRequest { request, error ->
94
+
if (error != null) {
95
+
println("Unable to perform the request: $error")
96
+
onProccessed(null)
97
+
} else {
98
+
request?.also { vnRequest ->
99
+
bodyPoseHandler(vnRequest).also { recognizedPoints ->
100
+
Skeleton(
101
+
timestamp = timestamp,
102
+
leftShoulder = recognizedPoints?.get(
103
+
VNHumanBodyPoseObservationJointNameLeftShoulder
104
+
)?.location?.toSkeletonPoint(width, height),
105
+
rightShoulder = recognizedPoints?.get(
106
+
VNHumanBodyPoseObservationJointNameRightShoulder
107
+
)?.location?.toSkeletonPoint(width, height),
108
+
leftElbow = recognizedPoints?.get(
109
+
VNHumanBodyPoseObservationJointNameLeftElbow
110
+
)?.location?.toSkeletonPoint(width, height),
111
+
rightElbow = recognizedPoints?.get(
112
+
VNHumanBodyPoseObservationJointNameRightElbow
113
+
)?.location?.toSkeletonPoint(width, height),
114
+
leftWrist = recognizedPoints?.get(
115
+
VNHumanBodyPoseObservationJointNameLeftWrist
116
+
)?.location?.toSkeletonPoint(width, height),
117
+
rightWrist = recognizedPoints?.get(
118
+
VNHumanBodyPoseObservationJointNameRightWrist
119
+
)?.location?.toSkeletonPoint(width, height),
120
+
leftHip = recognizedPoints?.get(
121
+
VNHumanBodyPoseObservationJointNameLeftHip
122
+
)?.location?.toSkeletonPoint(width, height),
123
+
rightHip = recognizedPoints?.get(
124
+
VNHumanBodyPoseObservationJointNameRightHip
125
+
)?.location?.toSkeletonPoint(width, height),
126
+
leftKnee = recognizedPoints?.get(
127
+
VNHumanBodyPoseObservationJointNameLeftKnee
128
+
)?.location?.toSkeletonPoint(width, height),
129
+
rightKnee = recognizedPoints?.get(
130
+
VNHumanBodyPoseObservationJointNameRightKnee
131
+
)?.location?.toSkeletonPoint(width, height),
132
+
leftAnkle = recognizedPoints?.get(
133
+
VNHumanBodyPoseObservationJointNameLeftAnkle
134
+
)?.location?.toSkeletonPoint(width, height),
135
+
rightAnkle = recognizedPoints?.get(
136
+
VNHumanBodyPoseObservationJointNameRightAnkle
137
+
)?.location?.toSkeletonPoint(width, height)
138
+
).also(onProccessed)
139
+
}
140
+
}
141
+
}
142
+
}.also { request ->
143
+
runCatching {
144
+
requestHandler.performRequests(
145
+
listOf(request), errorPtr.ptr
146
+
)
147
+
}
136
148
}
137
149
}
138
150
}
139
-
try {
140
-
val result = runCatching {
141
-
request(requestHandler,request,errorPtr)
142
-
}
143
-
144
-
if (result.isFailure) {
145
-
println("Exception during performRequests: ${result.exceptionOrNull()?.message}")
146
-
onProccessed(null)
147
-
return@memScoped
148
-
}
149
-
150
-
if (errorPtr.value != null) {
151
-
println("Error performing request: ${errorPtr.value}")
152
-
onProccessed(null)
153
-
}
154
-
} catch (e: Throwable) {
155
-
println("Unable to perform the request: ${e.message}")
156
-
onProccessed(null)
157
-
}
158
151
}
159
-
160
-
}
161
-
@OptIn(ExperimentalForeignApi::class)
162
-
@Throws(Throwable::class)
163
-
fun request(requestHandler: VNImageRequestHandler, request: VNRequest, errorPtr: ObjCObjectVar<NSError?>){
164
-
requestHandler.performRequests(listOf(request), errorPtr.ptr)
165
152
}
166
153
}
167
154
+3
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/MemoryManager.kt
+3
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/MemoryManager.kt
···
6
6
import platform.Foundation.NSOperationQueue
7
7
import platform.Foundation.NSProcessInfo
8
8
import platform.UIKit.UIApplicationDidReceiveMemoryWarningNotification
9
+
import kotlin.native.runtime.NativeRuntimeApi
9
10
10
11
/**
11
12
* Manages memory resources for camera operations
···
13
14
*/
14
15
object MemoryManager {
15
16
16
-
private const val MEMORY_PRESSURE_THRESHOLD = 0.8
17
+
private const val MEMORY_PRESSURE_THRESHOLD = 0.4
17
18
18
19
19
20
private val smallBufferLock = NSLock()
···
79
80
/**
80
81
* Handle high memory pressure situation
81
82
*/
83
+
@OptIn(NativeRuntimeApi::class)
82
84
private fun handleHighMemoryPressure() {
83
85
clearBufferPools()
84
86
}