This repository has no description
fix: improve ios stability
author
nathan holland
date
(Jun 26, 2025, 11:19 PM +0300)
commit
4e66247c
4e66247ccb54a7b787d1bc5376b8300300b5e257
parent
328d2c0c
328d2c0c9da7fd0c48bb4439bf155162b9027a90
+144
-12
+53
-11
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
+53
-11
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
···
10
10
import androidx.compose.ui.unit.LayoutDirection
11
11
import com.performancecoachlab.posedetection.skeleton.Skeleton
12
12
import com.performancecoachlab.posedetection.skeleton.SkeletonRepository
13
-
import kotlinx.cinterop.COpaquePointer
14
13
import kotlinx.cinterop.ExperimentalForeignApi
15
-
import kotlinx.cinterop.ObjCClass
16
14
import kotlinx.cinterop.addressOf
17
15
import kotlinx.cinterop.get
18
16
import kotlinx.cinterop.refTo
19
17
import kotlinx.cinterop.useContents
20
18
import kotlinx.cinterop.usePinned
21
-
import objcnames.classes.Protocol
22
19
import org.jetbrains.skia.ColorAlphaType
23
20
import org.jetbrains.skia.ColorSpace
24
21
import org.jetbrains.skia.ColorType
···
66
63
import platform.CoreGraphics.CGImageGetWidth
67
64
import platform.CoreGraphics.CGImageRelease
68
65
import platform.CoreGraphics.CGPointMake
69
-
import platform.CoreImage.CIImage
70
66
import platform.CoreMedia.CMSampleBufferGetImageBuffer
71
67
import platform.CoreMedia.CMSampleBufferRef
72
-
import platform.CoreVideo.CVPixelBufferGetHeight
73
-
import platform.CoreVideo.CVPixelBufferGetWidth
74
68
import platform.Foundation.NSData
75
69
import platform.Foundation.NSError
76
70
import platform.Foundation.NSURL
···
82
76
import platform.UIKit.UIViewController
83
77
import platform.darwin.DISPATCH_QUEUE_PRIORITY_HIGH
84
78
import platform.darwin.NSObject
85
-
import platform.darwin.NSUInteger
86
79
import platform.darwin.dispatch_async
87
80
import platform.darwin.dispatch_get_global_queue
88
81
import platform.darwin.dispatch_get_main_queue
···
434
427
didOutputSampleBuffer: CMSampleBufferRef?,
435
428
fromConnection: AVCaptureConnection
436
429
) {
430
+
timeStamp().also { timestamp->
431
+
runCatching {
432
+
dispatch_queue_create(
433
+
"com.performancecoachlab.frameProcessing", null
434
+
).also { processingQueue ->
435
+
dispatch_async(processingQueue) {
436
+
try {
437
+
frameProcessor.analyseBuffer(
438
+
CMSampleBufferGetImageBuffer(didOutputSampleBuffer), timestamp
439
+
) { skeleton ->
440
+
cameraPreviewLayer?.also { preview ->
441
+
skeleton?.let {
442
+
mapSkeletonToPreview(
443
+
skeleton = it,
444
+
previewLayer = preview,
445
+
isUsingFrontCamera = isUsingFrontCamera,
446
+
width = 480f,
447
+
height = 360f
448
+
)
449
+
}?.also { updatedSkeleton ->
450
+
skeletonRepository?.updateSkeleton(
451
+
updatedSkeleton
452
+
)
453
+
preview.bounds.useContents {
454
+
Pair(
455
+
size.width.toInt(),
456
+
size.height.toInt()
457
+
)
458
+
}.also { bo ->
459
+
ImageBitmap(
460
+
bo.first, bo.second
461
+
).drawSkeleton(inskeleton = updatedSkeleton)
462
+
.also { drawn ->
463
+
frameListener?.updateFrame(
464
+
drawn
465
+
)
466
+
}
467
+
}
468
+
}
469
+
}
470
+
}
471
+
}catch (e: Exception) {
472
+
println(e.message ?: "Unknown error in frame processing")
473
+
}
474
+
475
+
}
476
+
}
477
+
}
478
+
}
437
479
MemoryManager.updateMemoryStatus()
438
480
kotlin.native.runtime.GC.collect()
439
-
timeStamp().also { timestamp->
481
+
/* timeStamp().also { timestamp->
440
482
runCatching {
441
483
CMSampleBufferGetImageBuffer(didOutputSampleBuffer).also { buffer ->
442
484
CVPixelBufferGetWidth(buffer).also { w ->
···
477
519
drawn
478
520
)
479
521
// Add to image repository: use original skeleton and the raw frame image
480
-
/*imageRepository?.updateImage(
522
+
*//*imageRepository?.updateImage(
481
523
com.performancecoachlab.posedetection.recording.SkeletonImage(
482
524
skeleton = skeleton, // original, not mapped
483
525
image = safeBitmap
484
526
)
485
-
)*/
527
+
)*//*
486
528
}
487
529
}
488
530
}
···
497
539
}
498
540
}
499
541
}
500
-
}
542
+
}*/
501
543
}
502
544
fun ImageBitmap.copy(): ImageBitmap {
503
545
val original = this
+91
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
+91
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
···
12
12
import kotlinx.cinterop.ptr
13
13
import kotlinx.cinterop.useContents
14
14
import kotlinx.cinterop.value
15
+
import platform.AVFoundation.AVCaptureVideoOrientationLandscapeRight
15
16
import platform.CoreGraphics.CGImageGetHeight
16
17
import platform.CoreGraphics.CGImageGetWidth
17
18
import platform.CoreGraphics.CGImageRef
···
19
20
import platform.CoreGraphics.CGRect
20
21
import platform.CoreGraphics.CGRectMake
21
22
import platform.CoreML.MLModel
23
+
import platform.CoreVideo.CVImageBufferRef
22
24
import platform.CoreVideo.CVPixelBufferRef
23
25
import platform.Foundation.NSBundle
24
26
import platform.Foundation.NSError
···
41
43
import platform.Vision.VNImageRequestHandler
42
44
import platform.Vision.VNRecognizedPoint
43
45
import platform.Vision.VNRequest
44
-
import platform.Foundation.NSProcessInfo
45
46
import platform.Foundation.NSUUID
46
47
import platform.Vision.*
47
48
···
171
172
)
172
173
}
173
174
}
175
+
@OptIn(BetaInteropApi::class)
176
+
fun analyseBuffer(buffer: CVImageBufferRef?, timestamp: Long, onProcessed: (Skeleton?) -> Unit) {
177
+
if( buffer == null) {
178
+
onProcessed(null)
179
+
return
180
+
}
181
+
val width = 480uL
182
+
val height = 360uL
183
+
memScoped {
184
+
val errorPtr = alloc<ObjCObjectVar<NSError?>>()
185
+
val options = mapOf<Any?, Any?>(
186
+
"orientation" to AVCaptureVideoOrientationLandscapeRight
187
+
)
188
+
val requestHandler = VNImageRequestHandler(buffer, options)
189
+
val request = VNDetectHumanBodyPoseRequest { request, error ->
190
+
if (error != null) {
191
+
onProcessed(null)
192
+
} else {
193
+
request?.also { vnRequest ->
194
+
val recognizedPoints = bodyPoseHandler(vnRequest)
195
+
regionOfInterest = calculateRegionOfInterest(recognizedPoints)
196
+
val updatedSkeleton = Skeleton(
197
+
timestamp = timestamp,
198
+
leftShoulder = recognizedPoints?.get(
199
+
VNHumanBodyPoseObservationJointNameLeftShoulder
200
+
)?.location?.toSkeletonPoint(width, height),
201
+
rightShoulder = recognizedPoints?.get(
202
+
VNHumanBodyPoseObservationJointNameRightShoulder
203
+
)?.location?.toSkeletonPoint(width, height),
204
+
leftElbow = recognizedPoints?.get(
205
+
VNHumanBodyPoseObservationJointNameLeftElbow
206
+
)?.location?.toSkeletonPoint(width, height),
207
+
rightElbow = recognizedPoints?.get(
208
+
VNHumanBodyPoseObservationJointNameRightElbow
209
+
)?.location?.toSkeletonPoint(width, height),
210
+
leftWrist = recognizedPoints?.get(
211
+
VNHumanBodyPoseObservationJointNameLeftWrist
212
+
)?.location?.toSkeletonPoint(width, height),
213
+
rightWrist = recognizedPoints?.get(
214
+
VNHumanBodyPoseObservationJointNameRightWrist
215
+
)?.location?.toSkeletonPoint(width, height),
216
+
leftHip = recognizedPoints?.get(
217
+
VNHumanBodyPoseObservationJointNameLeftHip
218
+
)?.location?.toSkeletonPoint(width, height),
219
+
rightHip = recognizedPoints?.get(
220
+
VNHumanBodyPoseObservationJointNameRightHip
221
+
)?.location?.toSkeletonPoint(width, height),
222
+
leftKnee = recognizedPoints?.get(
223
+
VNHumanBodyPoseObservationJointNameLeftKnee
224
+
)?.location?.toSkeletonPoint(width, height),
225
+
rightKnee = recognizedPoints?.get(
226
+
VNHumanBodyPoseObservationJointNameRightKnee
227
+
)?.location?.toSkeletonPoint(width, height),
228
+
leftAnkle = recognizedPoints?.get(
229
+
VNHumanBodyPoseObservationJointNameLeftAnkle
230
+
)?.location?.toSkeletonPoint(width, height),
231
+
rightAnkle = recognizedPoints?.get(
232
+
VNHumanBodyPoseObservationJointNameRightAnkle
233
+
)?.location?.toSkeletonPoint(width, height),
234
+
height = height.toFloat(),
235
+
width = width.toFloat()
236
+
)
237
+
onProcessed(skelBuffer.smooth(updatedSkeleton))
238
+
}
239
+
}
240
+
}
241
+
request.regionOfInterest = regionOfInterest
242
+
try {
243
+
val result = runCatching {
244
+
request(requestHandler, request, errorPtr)
245
+
}
246
+
247
+
if (result.isFailure) {
248
+
println("Exception during performRequests: ${result.exceptionOrNull()?.message}")
249
+
onProcessed(null)
250
+
return@memScoped
251
+
}
252
+
253
+
if (errorPtr.value != null) {
254
+
println("Error performing request: ${errorPtr.value}")
255
+
onProcessed(null)
256
+
}
257
+
} catch (e: Throwable) {
258
+
println("Unable to perform the request: ${e.message}")
259
+
onProcessed(null)
260
+
}
261
+
}
262
+
}
174
263
175
264
@OptIn(BetaInteropApi::class)
176
265
fun analyseFrame(cgImage: CGImageRef?, timestamp: Long, onProccessed: (Skeleton?) -> Unit) {
···
181
270
}
182
271
val width = CGImageGetWidth(cgImage)
183
272
val height = CGImageGetHeight(cgImage)
273
+
print("width: $width, height: $height")
184
274
if (width.toUInt() == 0u || height.toUInt() == 0u) {
185
275
onProccessed(null)
186
276
return