This repository has no description
fix: handle memory issues
author
nathan holland
committer
Tangled
date
(May 2, 2025, 10:33 PM UTC)
commit
60791215
60791215a350f0cfc7ae42ebf337600f5ea5742c
parent
89200a15
89200a15818f0b8989b464907281b8cffb303913
+234
-165
+12
-2
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.kt
+12
-2
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.kt
···
10
10
import androidx.compose.ui.graphics.PaintingStyle.Companion.Fill
11
11
import androidx.compose.ui.graphics.PaintingStyle.Companion.Stroke
12
12
import androidx.compose.ui.graphics.drawscope.CanvasDrawScope
13
+
import androidx.compose.ui.graphics.drawscope.DrawStyle
14
+
import androidx.compose.ui.graphics.drawscope.Stroke
13
15
import androidx.compose.ui.graphics.rotate
16
+
import androidx.compose.ui.graphics.scale
14
17
import androidx.compose.ui.unit.Density
15
18
import androidx.compose.ui.unit.LayoutDirection
16
19
import com.performancecoachlab.posedetection.skeleton.Skeleton
17
20
18
-
19
-
fun ImageBitmap.drawSkeleton(skeleton: Skeleton?, rotation: Float = 0f): ImageBitmap {
21
+
fun ImageBitmap.drawSkeleton(inskeleton: Skeleton?, rotation: Float = 0f, mirrored: Boolean = false): ImageBitmap {
22
+
val skeleton = inskeleton?.let {
23
+
if (mirrored) {
24
+
it.mirror(this.height.toFloat())
25
+
} else {
26
+
it
27
+
}
28
+
}
20
29
also {
21
30
val bitmap = ImageBitmap(it.width, it.height)
22
31
val canvas = Canvas(bitmap).apply {
···
31
40
size,
32
41
) {
33
42
drawImage(it)
43
+
34
44
skeleton?.apply {
35
45
val paintWhite = Paint().apply {
36
46
color = Color.White
+18
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/skeleton/Skeleton.kt
+18
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/skeleton/Skeleton.kt
···
152
152
},
153
153
)
154
154
}
155
+
156
+
fun mirror(w:Float): Skeleton {
157
+
return Skeleton(
158
+
timestamp = timestamp,
159
+
leftShoulder = leftShoulder?.let { SkeletonCoordinate(it.x, w-it.y) },
160
+
rightShoulder = rightShoulder?.let { SkeletonCoordinate(it.x, w-it.y) },
161
+
leftElbow = leftElbow?.let { SkeletonCoordinate(it.x, w-it.y) },
162
+
rightElbow = rightElbow?.let { SkeletonCoordinate(it.x, w-it.y) },
163
+
leftWrist = leftWrist?.let { SkeletonCoordinate(it.x, w-it.y) },
164
+
rightWrist = rightWrist?.let { SkeletonCoordinate(it.x, w-it.y) },
165
+
leftHip = leftHip?.let { SkeletonCoordinate(it.x,w-it.y) },
166
+
rightHip = rightHip?.let { SkeletonCoordinate(it.x, w-it.y) },
167
+
leftKnee = leftKnee?.let { SkeletonCoordinate(it.x, w-it.y) },
168
+
rightKnee = rightKnee?.let { SkeletonCoordinate(it.x, w-it.y) },
169
+
leftAnkle = leftAnkle?.let { SkeletonCoordinate(it.x, w-it.y) },
170
+
rightAnkle = rightAnkle?.let { SkeletonCoordinate(it.x, w-it.y) },
171
+
)
172
+
}
155
173
}
+92
-16
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
+92
-16
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
···
8
8
import androidx.compose.ui.graphics.toComposeImageBitmap
9
9
import androidx.compose.ui.unit.Density
10
10
import androidx.compose.ui.unit.LayoutDirection
11
+
import com.performancecoachlab.posedetection.skeleton.Skeleton
11
12
import com.performancecoachlab.posedetection.skeleton.SkeletonRepository
12
13
import kotlinx.atomicfu.atomic
13
14
import kotlinx.cinterop.ExperimentalForeignApi
14
15
import kotlinx.cinterop.addressOf
15
16
import kotlinx.cinterop.get
16
17
import kotlinx.cinterop.refTo
18
+
import kotlinx.cinterop.useContents
17
19
import kotlinx.cinterop.usePinned
18
20
import kotlinx.coroutines.CoroutineScope
19
21
import kotlinx.coroutines.Dispatchers
···
33
35
import platform.AVFoundation.AVCaptureInput
34
36
import platform.AVFoundation.AVCaptureOutput
35
37
import platform.AVFoundation.AVCaptureSession
36
-
import platform.AVFoundation.AVCaptureSessionPreset640x480
38
+
import platform.AVFoundation.AVCaptureSessionPresetMedium
37
39
import platform.AVFoundation.AVCaptureVideoDataOutput
38
40
import platform.AVFoundation.AVCaptureVideoDataOutputSampleBufferDelegateProtocol
39
41
import platform.AVFoundation.AVCaptureVideoOrientation
···
42
44
import platform.AVFoundation.AVCaptureVideoOrientationPortrait
43
45
import platform.AVFoundation.AVCaptureVideoOrientationPortraitUpsideDown
44
46
import platform.AVFoundation.AVCaptureVideoPreviewLayer
47
+
import platform.AVFoundation.AVLayerVideoGravityResize
45
48
import platform.AVFoundation.AVLayerVideoGravityResizeAspectFill
46
49
import platform.AVFoundation.AVMediaTypeVideo
47
50
import platform.AVFoundation.AVVideoCodecJPEG
···
62
65
import platform.CoreGraphics.CGImageGetHeight
63
66
import platform.CoreGraphics.CGImageGetWidth
64
67
import platform.CoreGraphics.CGImageRelease
68
+
import platform.CoreGraphics.CGPointMake
65
69
import platform.CoreImage.CIImage
66
70
import platform.CoreMedia.CMSampleBufferGetImageBuffer
67
71
import platform.CoreMedia.CMSampleBufferRef
72
+
import platform.CoreVideo.CVPixelBufferGetHeight
68
73
import platform.CoreVideo.CVPixelBufferGetWidth
69
74
import platform.Foundation.NSData
70
75
import platform.UIKit.UIDevice
···
77
82
import platform.darwin.dispatch_async
78
83
import platform.darwin.dispatch_get_global_queue
79
84
import platform.darwin.dispatch_get_main_queue
85
+
import platform.darwin.dispatch_queue_create
80
86
import platform.posix.memcpy
81
87
import kotlin.native.runtime.NativeRuntimeApi
82
88
83
89
class CameraEngine : UIViewController(null, null) {
84
90
private var isCapturing = atomic(false)
85
91
val cameraController = CameraController()
86
-
private var frameListener: FrameRepository? = null
87
-
88
-
89
-
fun addFrameListener(listener: FrameRepository) {
90
-
frameListener = listener
91
-
}
92
92
93
93
private val memoryManager = MemoryManager
94
94
···
157
157
override fun viewDidLayoutSubviews() {
158
158
super.viewDidLayoutSubviews()
159
159
cameraController.cameraPreviewLayer?.setFrame(view.bounds)
160
+
cameraController.cameraPreviewLayer?.contentsGravity = AVLayerVideoGravityResizeAspectFill
160
161
}
161
162
162
163
fun normaliseImageRotation(image: NSData): ImageBitmap {
···
187
188
fun addSkeletonRepository(repository: SkeletonRepository) {
188
189
cameraController.skeletonRepository = repository
189
190
}
191
+
192
+
fun addFrameListener(listener: FrameRepository) {
193
+
cameraController.frameListener = listener
194
+
}
190
195
}
191
196
192
197
class CameraController : NSObject(), AVCaptureVideoDataOutputSampleBufferDelegateProtocol {
···
198
203
var cameraPreviewLayer: AVCaptureVideoPreviewLayer? = null
199
204
var isUsingFrontCamera = true
200
205
var skeletonRepository: SkeletonRepository? = null
206
+
var frameListener: FrameRepository? = null
201
207
var onError: ((CameraException) -> Unit)? = null
202
208
203
209
sealed class CameraException : Exception() {
···
210
216
try {
211
217
captureSession = AVCaptureSession()
212
218
captureSession?.beginConfiguration()
213
-
captureSession?.sessionPreset = AVCaptureSessionPreset640x480
219
+
captureSession?.sessionPreset = AVCaptureSessionPresetMedium
214
220
215
221
if (!setupInputs()) {
216
222
throw CameraException.DeviceNotAvailable()
···
381
387
fromConnection: AVCaptureConnection
382
388
) {
383
389
val timestamp = timeStamp()
384
-
println("$timestamp x")
385
390
if (isProcessing.compareAndSet(expect = false, update = true)) {
386
391
try {
392
+
val result = runCatching {
393
+
387
394
val buffer = CMSampleBufferGetImageBuffer(didOutputSampleBuffer)
388
395
val w = CVPixelBufferGetWidth(buffer)
389
-
val h = CVPixelBufferGetWidth(buffer)
396
+
val h = CVPixelBufferGetHeight(buffer)
390
397
val imageBuffer = CIImage.imageWithCVPixelBuffer(buffer)
391
398
val cgImage = imageBuffer.toCGImageRef(w, h)
392
-
frameProcessor.processPose(cgImage, timestamp) { updatedSkeleton ->
393
-
updatedSkeleton?.also { skel ->
394
-
skeletonRepository?.updateSkeleton(skel)
399
+
val processingQueue = dispatch_queue_create("com.performancecoachlab.frameProcessing", null)
400
+
dispatch_async(processingQueue) {
401
+
frameProcessor.processPose(cgImage, timestamp) { skeleton ->
402
+
cameraPreviewLayer?.also { preview->
403
+
404
+
val updatedSkeleton = skeleton?.let {
405
+
mapSkeletonToPreview(
406
+
skeleton = it,
407
+
previewLayer = preview,
408
+
isUsingFrontCamera = isUsingFrontCamera,
409
+
width = w.toFloat(),
410
+
height = h.toFloat()
411
+
)
412
+
}
413
+
updatedSkeleton?.also { skel ->
414
+
skeletonRepository?.updateSkeleton(skel)
415
+
}
416
+
val bo = preview.bounds.useContents { Pair(size.width.toInt(), size.height.toInt()) }
417
+
val drawn = ImageBitmap(bo.first,bo.second).drawSkeleton(inskeleton = updatedSkeleton)
418
+
frameListener?.updateFrame(drawn)
419
+
}
420
+
CGImageRelease(cgImage)
395
421
}
396
422
}
397
-
CGImageRelease(cgImage)
398
-
kotlin.native.runtime.GC.collect()
423
+
}
424
+
425
+
if (result.isFailure) {
426
+
println("Exception during performRequests: ${result.exceptionOrNull()?.message}")
427
+
}
399
428
} catch (e: Exception) {
400
429
println("Error during frame processing: ${e.message}")
401
-
} finally {
430
+
} catch (e: Throwable){
431
+
println("Error during frame processing: ${e.message}")
432
+
}finally {
402
433
isProcessing.value = false
434
+
kotlin.native.runtime.GC.collect()
403
435
}
404
436
}
405
437
}
406
438
407
439
private inline fun guard(condition: Boolean, crossinline block: () -> Unit) {
408
440
if (!condition) block()
441
+
}
442
+
443
+
444
+
private fun ImageBitmap.normaliseRotation(): ImageBitmap {
445
+
return when (UIDevice.currentDevice.orientation) {
446
+
UIDeviceOrientation.UIDeviceOrientationPortrait -> this.rotateLeft()
447
+
UIDeviceOrientation.UIDeviceOrientationPortraitUpsideDown -> this.rotateRight()
448
+
UIDeviceOrientation.UIDeviceOrientationLandscapeLeft -> if (!isUsingFrontCamera) this.rotate180() else this
449
+
else -> if (!isUsingFrontCamera) this else this.rotate180()
450
+
}
451
+
}
452
+
453
+
@OptIn(ExperimentalForeignApi::class)
454
+
fun mapSkeletonToPreview(
455
+
skeleton: Skeleton,
456
+
previewLayer: AVCaptureVideoPreviewLayer,
457
+
isUsingFrontCamera: Boolean,
458
+
width: Float,
459
+
height: Float
460
+
): Skeleton {
461
+
fun mapPoint(point: Skeleton.SkeletonCoordinate?): Skeleton.SkeletonCoordinate? {
462
+
if (point == null) return null
463
+
464
+
// Normalize the point
465
+
val normalizedPoint = CGPointMake(point.x.toDouble()/width, point.y.toDouble()/height)
466
+
val screenPoint = previewLayer.pointForCaptureDevicePointOfInterest(normalizedPoint)
467
+
return Skeleton.SkeletonCoordinate(screenPoint.useContents { x.toFloat() }, screenPoint.useContents { y.toFloat() })
468
+
}
469
+
470
+
return Skeleton(
471
+
timestamp = skeleton.timestamp,
472
+
leftShoulder = mapPoint(skeleton.leftShoulder),
473
+
rightShoulder = mapPoint(skeleton.rightShoulder),
474
+
leftElbow = mapPoint(skeleton.leftElbow),
475
+
rightElbow = mapPoint(skeleton.rightElbow),
476
+
leftWrist = mapPoint(skeleton.leftWrist),
477
+
rightWrist = mapPoint(skeleton.rightWrist),
478
+
leftHip = mapPoint(skeleton.leftHip),
479
+
rightHip = mapPoint(skeleton.rightHip),
480
+
leftKnee = mapPoint(skeleton.leftKnee),
481
+
rightKnee = mapPoint(skeleton.rightKnee),
482
+
leftAnkle = mapPoint(skeleton.leftAnkle),
483
+
rightAnkle = mapPoint(skeleton.rightAnkle)
484
+
)
409
485
}
410
486
411
487
}
+22
-87
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt
+22
-87
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt
···
3
3
import androidx.compose.foundation.Image
4
4
import androidx.compose.foundation.layout.Box
5
5
import androidx.compose.foundation.layout.fillMaxSize
6
+
import androidx.compose.foundation.layout.fillMaxWidth
6
7
import androidx.compose.foundation.layout.height
8
+
import androidx.compose.foundation.layout.requiredHeight
9
+
import androidx.compose.foundation.layout.requiredWidth
7
10
import androidx.compose.foundation.layout.width
8
11
import androidx.compose.material3.Surface
9
12
import androidx.compose.material3.Text
···
54
57
import platform.Vision.VNImageRequestHandler
55
58
import platform.Vision.VNRecognizedPoint
56
59
import platform.Vision.VNRequest
60
+
import kotlin.native.runtime.NativeRuntimeApi
57
61
import kotlin.time.Clock
58
62
import kotlin.time.ExperimentalTime
59
63
64
+
@OptIn(NativeRuntimeApi::class)
60
65
@Composable
61
66
actual fun CameraView(
62
67
skeletonRepository: SkeletonRepository,
···
65
70
) {
66
71
val cameraEngine = remember { mutableStateOf<CameraEngine?>(null) }
67
72
val skeleton by skeletonRepository.skeletonFlow.collectAsState()
73
+
val frameListener = remember { FrameRepository() }
74
+
val frameBitmap by frameListener.frameFlow.collectAsState()
68
75
LaunchedEffect(cameraEngine.value){
69
-
cameraEngine.value?.addSkeletonRepository(skeletonRepository)
76
+
cameraEngine.value?.apply {
77
+
addSkeletonRepository(skeletonRepository)
78
+
addFrameListener(frameListener)
79
+
}
80
+
70
81
}
71
82
Box(modifier = Modifier.fillMaxSize()) {
72
83
CameraPreview(
···
74
85
onCameraControllerReady = { engine ->
75
86
cameraEngine.value = engine
76
87
})
77
-
skeleton.also {
78
-
Text(it.toString())
79
-
}
80
-
}
81
-
}
82
-
83
-
@OptIn(ExperimentalTime::class, ExperimentalForeignApi::class)
84
-
@Composable
85
-
fun DetectPoseView(
86
-
cameraEngine: CameraEngine,
87
-
skeletonRepository: SkeletonRepository,
88
-
drawSkeleton: Boolean,
89
-
modifier: Modifier
90
-
) {
91
-
val scope = rememberCoroutineScope()
92
-
93
-
val frameRepository = remember { FrameRepository() }
94
-
val frameBitmap by frameRepository.frameFlow.collectAsState()
95
-
96
-
val skeleton by skeletonRepository.skeletonFlow.collectAsState()
97
-
98
-
LaunchedEffect(cameraEngine){
99
-
cameraEngine.addFrameListener(frameRepository)
100
-
cameraEngine.addSkeletonRepository(skeletonRepository)
101
-
}
102
-
103
-
/*LaunchedEffect(Unit){
104
-
scope.launch {
105
-
while(true){
106
-
frameBitmap?.also {
107
-
println("FrameBitmap not null")
108
-
it.processPose(
109
-
Clock.System.now().toEpochMilliseconds(),
110
-
skeletonRepository
111
-
)
112
-
imageBitmap = ImageBitmap(640,640).drawSkeleton(
113
-
if (drawSkeleton) skeletonRepository.skeletonFlow.value else null,
114
-
0f
115
-
)
116
-
}?: delay(50L)
88
+
if (drawSkeleton) {
89
+
frameBitmap?.also {
90
+
Image(
91
+
bitmap = it,
92
+
contentDescription = "video frame",
93
+
modifier = Modifier.fillMaxSize(),
94
+
contentScale = ContentScale.Crop,
95
+
)
96
+
kotlin.native.runtime.GC.collect()
117
97
}
118
98
}
119
-
}*/
120
99
121
-
122
-
/*LaunchedEffect(cameraEngine) {
123
-
scope.launch {
124
-
while (true) {
125
-
cameraEngine.getFrame().also { result ->
126
-
if (result is FrameCaptureResult.Success) {
127
-
result.imageBitmap?.also {
128
-
it.processPose(
129
-
Clock.System.now().toEpochMilliseconds(),
130
-
skeletonRepository
131
-
)
132
-
imageBitmap = it.drawSkeleton(
133
-
if (drawSkeleton) skeletonRepository.skeletonFlow.value else null,
134
-
0f
135
-
)
136
-
}
137
-
} else if (result is FrameCaptureResult.Error) {
138
-
println("Error capturing frame: ${result.exception}")
139
-
}
140
-
}
141
-
}
142
-
}
143
-
}*/
144
-
//SkeletonBitmap(skeleton)
145
-
}
146
-
147
-
@Composable
148
-
fun SkeletonBitmap(skeleton: Skeleton?) {
149
-
var imageBitmap by remember { mutableStateOf<ImageBitmap?>(null) }
150
-
SideEffect {
151
-
skeleton?.also {
152
-
imageBitmap = ImageBitmap(640,640).drawSkeleton(
153
-
it,
154
-
90f
155
-
)
156
-
}
157
100
}
158
-
imageBitmap?.also {
159
-
Image(
160
-
bitmap = it,
161
-
contentDescription = "video frame",
162
-
contentScale = ContentScale.Crop,
163
-
)
164
-
}
165
-
}
166
-
101
+
}
+1
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/Conversions.kt
+1
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/Conversions.kt
+86
-56
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
+86
-56
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
···
2
2
3
3
import com.performancecoachlab.posedetection.skeleton.Skeleton
4
4
import com.performancecoachlab.posedetection.skeleton.SkeletonRepository
5
+
import kotlinx.cinterop.BetaInteropApi
5
6
import kotlinx.cinterop.CValue
6
7
import kotlinx.cinterop.ExperimentalForeignApi
8
+
import kotlinx.cinterop.ObjCObjectVar
9
+
import kotlinx.cinterop.alloc
10
+
import kotlinx.cinterop.memScoped
11
+
import kotlinx.cinterop.ptr
7
12
import kotlinx.cinterop.useContents
8
13
import kotlinx.cinterop.usePinned
14
+
import kotlinx.cinterop.value
9
15
import platform.CoreGraphics.CGImageGetHeight
10
16
import platform.CoreGraphics.CGImageGetWidth
11
17
import platform.CoreGraphics.CGImageRef
12
18
import platform.CoreGraphics.CGPoint
13
19
import platform.CoreMedia.CMSampleBufferRef
20
+
import platform.Foundation.NSError
21
+
import platform.UIKit.UIImage
14
22
import platform.Vision.VNDetectHumanBodyPoseRequest
15
23
import platform.Vision.VNHumanBodyPoseObservation
16
24
import platform.Vision.VNHumanBodyPoseObservationJointName
···
32
40
import platform.Vision.VNRequest
33
41
import platform.posix.size_t
34
42
import platform.zlib.uLong
43
+
import kotlin.native.runtime.NativeRuntimeApi
35
44
36
45
class FrameProcessor {
37
46
fun bodyPoseHandler(request: VNRequest): MutableMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint>? {
···
56
65
}
57
66
}
58
67
}
59
-
println("observation done")
60
68
return points
61
69
}
62
70
···
69
77
}
70
78
}
71
79
72
-
@OptIn(ExperimentalForeignApi::class)
80
+
@OptIn(ExperimentalForeignApi::class, NativeRuntimeApi::class, BetaInteropApi::class)
73
81
fun processPose(cgImage: CGImageRef?, timestamp: Long, onProccessed: (Skeleton?) -> Unit) {
74
82
if (cgImage == null) {
75
83
onProccessed(null)
···
81
89
onProccessed(null)
82
90
return
83
91
}
84
-
val requestHandler = VNImageRequestHandler(cgImage, mapOf<Any?, Any?>())
85
-
val request = VNDetectHumanBodyPoseRequest { request, error ->
86
-
if (error != null) {
87
-
println("Unable to perform the request: $error")
88
-
onProccessed(null)
89
-
} else {
90
-
request?.also { vnRequest ->
91
-
val recognizedPoints = bodyPoseHandler(vnRequest)
92
-
val updatedSkeleton = Skeleton(
93
-
timestamp = timestamp,
94
-
leftShoulder = recognizedPoints?.get(
95
-
VNHumanBodyPoseObservationJointNameLeftShoulder
96
-
)?.location?.toSkeletonPoint(width, height),
97
-
rightShoulder = recognizedPoints?.get(
98
-
VNHumanBodyPoseObservationJointNameRightShoulder
99
-
)?.location?.toSkeletonPoint(width, height),
100
-
leftElbow = recognizedPoints?.get(
101
-
VNHumanBodyPoseObservationJointNameLeftElbow
102
-
)?.location?.toSkeletonPoint(width, height),
103
-
rightElbow = recognizedPoints?.get(
104
-
VNHumanBodyPoseObservationJointNameRightElbow
105
-
)?.location?.toSkeletonPoint(width, height),
106
-
leftWrist = recognizedPoints?.get(
107
-
VNHumanBodyPoseObservationJointNameLeftWrist
108
-
)?.location?.toSkeletonPoint(width, height),
109
-
rightWrist = recognizedPoints?.get(
110
-
VNHumanBodyPoseObservationJointNameRightWrist
111
-
)?.location?.toSkeletonPoint(width, height),
112
-
leftHip = recognizedPoints?.get(
113
-
VNHumanBodyPoseObservationJointNameLeftHip
114
-
)?.location?.toSkeletonPoint(width, height),
115
-
rightHip = recognizedPoints?.get(
116
-
VNHumanBodyPoseObservationJointNameRightHip
117
-
)?.location?.toSkeletonPoint(width, height),
118
-
leftKnee = recognizedPoints?.get(
119
-
VNHumanBodyPoseObservationJointNameLeftKnee
120
-
)?.location?.toSkeletonPoint(width, height),
121
-
rightKnee = recognizedPoints?.get(
122
-
VNHumanBodyPoseObservationJointNameRightKnee
123
-
)?.location?.toSkeletonPoint(width, height),
124
-
leftAnkle = recognizedPoints?.get(
125
-
VNHumanBodyPoseObservationJointNameLeftAnkle
126
-
)?.location?.toSkeletonPoint(width, height),
127
-
rightAnkle = recognizedPoints?.get(
128
-
VNHumanBodyPoseObservationJointNameRightAnkle
129
-
)?.location?.toSkeletonPoint(width, height)
130
-
)
131
-
onProccessed(updatedSkeleton)
92
+
memScoped {
93
+
val errorPtr = alloc<ObjCObjectVar<NSError?>>()
94
+
val requestHandler = VNImageRequestHandler(cgImage, mapOf<Any?, Any?>())
95
+
val request = VNDetectHumanBodyPoseRequest { request, error ->
96
+
if (error != null) {
97
+
println("Unable to perform the request: $error")
98
+
onProccessed(null)
99
+
} else {
100
+
request?.also { vnRequest ->
101
+
val recognizedPoints = bodyPoseHandler(vnRequest)
102
+
val updatedSkeleton = Skeleton(
103
+
timestamp = timestamp,
104
+
leftShoulder = recognizedPoints?.get(
105
+
VNHumanBodyPoseObservationJointNameLeftShoulder
106
+
)?.location?.toSkeletonPoint(width, height),
107
+
rightShoulder = recognizedPoints?.get(
108
+
VNHumanBodyPoseObservationJointNameRightShoulder
109
+
)?.location?.toSkeletonPoint(width, height),
110
+
leftElbow = recognizedPoints?.get(
111
+
VNHumanBodyPoseObservationJointNameLeftElbow
112
+
)?.location?.toSkeletonPoint(width, height),
113
+
rightElbow = recognizedPoints?.get(
114
+
VNHumanBodyPoseObservationJointNameRightElbow
115
+
)?.location?.toSkeletonPoint(width, height),
116
+
leftWrist = recognizedPoints?.get(
117
+
VNHumanBodyPoseObservationJointNameLeftWrist
118
+
)?.location?.toSkeletonPoint(width, height),
119
+
rightWrist = recognizedPoints?.get(
120
+
VNHumanBodyPoseObservationJointNameRightWrist
121
+
)?.location?.toSkeletonPoint(width, height),
122
+
leftHip = recognizedPoints?.get(
123
+
VNHumanBodyPoseObservationJointNameLeftHip
124
+
)?.location?.toSkeletonPoint(width, height),
125
+
rightHip = recognizedPoints?.get(
126
+
VNHumanBodyPoseObservationJointNameRightHip
127
+
)?.location?.toSkeletonPoint(width, height),
128
+
leftKnee = recognizedPoints?.get(
129
+
VNHumanBodyPoseObservationJointNameLeftKnee
130
+
)?.location?.toSkeletonPoint(width, height),
131
+
rightKnee = recognizedPoints?.get(
132
+
VNHumanBodyPoseObservationJointNameRightKnee
133
+
)?.location?.toSkeletonPoint(width, height),
134
+
leftAnkle = recognizedPoints?.get(
135
+
VNHumanBodyPoseObservationJointNameLeftAnkle
136
+
)?.location?.toSkeletonPoint(width, height),
137
+
rightAnkle = recognizedPoints?.get(
138
+
VNHumanBodyPoseObservationJointNameRightAnkle
139
+
)?.location?.toSkeletonPoint(width, height)
140
+
)
141
+
onProccessed(updatedSkeleton)
142
+
}
132
143
}
133
144
}
145
+
try {
146
+
val result = runCatching {
147
+
request(requestHandler,request,errorPtr)
148
+
}
149
+
150
+
if (result.isFailure) {
151
+
println("Exception during performRequests: ${result.exceptionOrNull()?.message}")
152
+
onProccessed(null)
153
+
return@memScoped
154
+
}
155
+
156
+
if (errorPtr.value != null) {
157
+
println("Error performing request: ${errorPtr.value}")
158
+
onProccessed(null)
159
+
}
160
+
} catch (e: Throwable) {
161
+
println("Unable to perform the request: ${e.message}")
162
+
onProccessed(null)
163
+
}
134
164
}
135
-
try {
136
-
requestHandler.performRequests(listOf(request), null)
137
-
} catch (e: Exception) {
138
-
println("Unable to perform the request: ${e.message}")
139
-
onProccessed(null)
140
-
}
165
+
166
+
}
167
+
@OptIn(ExperimentalForeignApi::class)
168
+
@Throws(Throwable::class)
169
+
fun request(requestHandler: VNImageRequestHandler, request: VNRequest, errorPtr: ObjCObjectVar<NSError?>){
170
+
requestHandler.performRequests(listOf(request), errorPtr.ptr)
141
171
}
142
172
}
143
173
+3
-3
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameRepository.kt
+3
-3
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameRepository.kt
···
8
8
9
9
class FrameRepository {
10
10
@OptIn(ExperimentalForeignApi::class)
11
-
private val _frameFlow = MutableStateFlow<CGImageRef?>(null)
11
+
private val _frameFlow = MutableStateFlow<ImageBitmap?>(null)
12
12
@OptIn(ExperimentalForeignApi::class)
13
-
val frameFlow: StateFlow<CGImageRef?> get() = _frameFlow
13
+
val frameFlow: StateFlow<ImageBitmap?> get() = _frameFlow
14
14
15
15
@OptIn(ExperimentalForeignApi::class)
16
-
fun updateFrame(frame: CGImageRef) {
16
+
fun updateFrame(frame: ImageBitmap) {
17
17
_frameFlow.value = frame
18
18
}
19
19
}