This repository has no description
feat: draw objects in camera overlay on ios
author
nathan holland
date
(Jul 20, 2025, 5:06 PM +0100)
commit
87da3bc3
87da3bc33811072671ab16308a4f6f366f7b4793
parent
4f624d9c
4f624d9ce4b9407ad38ecb2259839a5d9307795e
+298
-438
+1
-1
posedetection/build.gradle.kts
+1
-1
posedetection/build.gradle.kts
···
89
89
implementation(libs.pose.detection)
90
90
implementation(libs.pose.detection.common)
91
91
implementation(libs.androidx.media3.common.ktx)
92
-
implementation("com.google.mlkit:object-detection-custom:17.0.2")
93
92
implementation ("org.tensorflow:tensorflow-lite-support:0.4.4")
93
+
implementation("org.tensorflow:tensorflow-lite-task-vision:0.4.4")
94
94
}
95
95
96
96
}
+25
-27
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
+25
-27
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
···
43
43
import androidx.camera.lifecycle.ProcessCameraProvider
44
44
import androidx.camera.core.Preview
45
45
import androidx.camera.core.ImageAnalysis
46
+
import androidx.camera.core.ImageProxy
46
47
import androidx.compose.runtime.rememberCoroutineScope
47
48
import androidx.compose.ui.geometry.Rect
48
-
import com.performancecoachlab.posedetection.custom.CustomObjectModel
49
+
import com.google.mlkit.vision.pose.PoseDetector
50
+
import com.performancecoachlab.posedetection.custom.CustomObjectDetectorModels
49
51
import com.performancecoachlab.posedetection.custom.CustomObjectRespository
50
-
import com.performancecoachlab.posedetection.objects.createObjectDetector
51
52
import com.performancecoachlab.posedetection.recording.AnalysisObject
52
53
import com.performancecoachlab.posedetection.recording.AnalysisResult
53
54
import kotlinx.coroutines.launch
55
+
import org.tensorflow.lite.support.image.TensorImage
56
+
import org.tensorflow.lite.task.vision.detector.ObjectDetector
54
57
55
58
@OptIn(ExperimentalGetImage::class)
56
59
@Composable
···
65
68
onVideoSaved: (String) -> Unit,
66
69
) {
67
70
var bitmap by remember { mutableStateOf<ImageBitmap?>(null) }
68
-
var skeleton by remember { mutableStateOf(Skeleton(width = 0f, height = 0f)) }
69
-
var objectsDetected by remember { mutableStateOf<List<AnalysisObject>>(emptyList()) }
70
71
val options =
71
72
PoseDetectorOptions.Builder().setDetectorMode(PoseDetectorOptions.STREAM_MODE).build()
72
73
val poseDetector = PoseDetection.getClient(options)
···
74
75
val lifecycleOwner: LifecycleOwner = LocalLifecycleOwner.current
75
76
val previewView: PreviewView = remember { PreviewView(context) }
76
77
val executor = remember { Executors.newSingleThreadExecutor() }
77
-
val modelPath = CustomObjectModel.getInstance().androidModelPath
78
-
val objectDetector = createObjectDetector(modelPath)
78
+
val objDetectors = CustomObjectDetectorModels.getInstance().models
79
79
val scope = rememberCoroutineScope()
80
80
var firstFrameTimestamp: Long? = null
81
81
···
135
135
val imageAnalysis = ImageAnalysis.Builder().build().also { analysis ->
136
136
analysis.setAnalyzer(executor) { imageProxy ->
137
137
val timestamp = System.currentTimeMillis()
138
-
imageProxy.image?.let { image ->
139
-
val img = InputImage.fromMediaImage(
140
-
image, imageProxy.imageInfo.rotationDegrees
141
-
)
142
-
img.process(objectDetector, poseDetector,timestamp){
143
-
customObjectRepository.updateCustomObject(it.objects)
144
-
it.skeleton?.let { skel ->
145
-
skeletonRepository.updateSkeleton(skel.mirror())
146
-
}
147
-
bitmap =
148
-
imageProxy.toBitmap().rotate(imageProxy.imageInfo.rotationDegrees.toFloat())
149
-
.asImageBitmap().let { inbmp ->
150
-
addFrame(inbmp,timestamp)
151
-
if( drawSkeleton) {
152
-
inbmp.drawAnalysisResults(it)
153
-
}else{
154
-
inbmp.drawSkeleton(null)
155
-
}
138
+
imageProxy.process(
139
+
objDetectors.firstOrNull()?.getDetector(), poseDetector, timestamp
140
+
){
141
+
customObjectRepository.updateCustomObject(it.objects)
142
+
it.skeleton?.let { skel ->
143
+
skeletonRepository.updateSkeleton(skel.mirror())
144
+
}
145
+
bitmap =
146
+
imageProxy.toBitmap().rotate(imageProxy.imageInfo.rotationDegrees.toFloat())
147
+
.asImageBitmap().let { inbmp ->
148
+
addFrame(inbmp,timestamp)
149
+
if( drawSkeleton) {
150
+
inbmp.drawAnalysisResults(it)
151
+
}else{
152
+
inbmp.drawSkeleton(null)
156
153
}
157
-
imageProxy.close()
158
-
}
159
-
} ?: imageProxy.close()
154
+
}
155
+
imageProxy.close()
156
+
}
160
157
}
161
158
}
162
159
cameraProvider.bindToLifecycle(
···
195
192
}
196
193
}
197
194
}
195
+
198
196
199
197
200
198
private fun PoseLandmark?.toSkeletonCoords(): Skeleton.SkeletonCoordinate? {
+82
-29
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.android.kt
+82
-29
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.android.kt
···
1
1
package com.performancecoachlab.posedetection.camera
2
2
3
+
import android.graphics.Bitmap
4
+
import androidx.annotation.OptIn
5
+
import androidx.camera.core.ExperimentalGetImage
6
+
import androidx.camera.core.ImageProxy
3
7
import androidx.compose.runtime.getValue
4
8
import androidx.compose.runtime.mutableStateOf
5
9
import androidx.compose.runtime.remember
6
10
import androidx.compose.runtime.setValue
7
11
import androidx.compose.ui.geometry.Rect
8
12
import androidx.compose.ui.graphics.asImageBitmap
13
+
import androidx.core.graphics.toRect
9
14
import com.google.android.gms.tasks.Tasks
10
15
import com.google.mlkit.vision.common.InputImage
11
16
import com.google.mlkit.vision.objects.DetectedObject.Label
···
14
19
import com.performancecoachlab.posedetection.recording.AnalysisObject
15
20
import com.performancecoachlab.posedetection.recording.AnalysisResult
16
21
import com.performancecoachlab.posedetection.skeleton.Skeleton
22
+
import org.tensorflow.lite.support.image.TensorImage
17
23
18
24
actual enum class PlatformType {
19
25
ANDROID, IOS;
···
24
30
25
31
}
26
32
27
-
fun InputImage.process(
28
-
objectDetector: ObjectDetector?,
33
+
34
+
@OptIn(ExperimentalGetImage::class)
35
+
fun ImageProxy.process(
36
+
objectDetector: org.tensorflow.lite.task.vision.detector.ObjectDetector?,
37
+
poseDetector: PoseDetector,
38
+
timestamp: Long,
39
+
onComplete: (AnalysisResult) -> Unit
40
+
) {
41
+
val tensorImage = TensorImage.fromBitmap(toBitmap())
42
+
val mlKitImage = image?.let{
43
+
InputImage.fromMediaImage(
44
+
it, imageInfo.rotationDegrees
45
+
)
46
+
}
47
+
process(
48
+
tensorImage = tensorImage,
49
+
mlKitImage = mlKitImage,
50
+
objectDetector = objectDetector,
51
+
poseDetector = poseDetector,
52
+
timestamp = timestamp,
53
+
width = width,
54
+
height = height,
55
+
onComplete = onComplete
56
+
)
57
+
}
58
+
59
+
fun Bitmap.process(
60
+
objectDetector: org.tensorflow.lite.task.vision.detector.ObjectDetector?,
61
+
poseDetector: PoseDetector,
62
+
timestamp: Long,
63
+
onComplete: (AnalysisResult) -> Unit
64
+
) {
65
+
val tensorImage = TensorImage.fromBitmap(this)
66
+
val mlKitImage = InputImage.fromBitmap(this, 0)
67
+
process(
68
+
tensorImage = tensorImage,
69
+
mlKitImage = mlKitImage,
70
+
objectDetector = objectDetector,
71
+
poseDetector = poseDetector,
72
+
timestamp = timestamp,
73
+
width = width,
74
+
height = height,
75
+
onComplete = onComplete
76
+
)
77
+
}
78
+
79
+
private fun process (
80
+
tensorImage: TensorImage,
81
+
mlKitImage: InputImage?,
82
+
objectDetector: org.tensorflow.lite.task.vision.detector.ObjectDetector?,
29
83
poseDetector: PoseDetector,
30
84
timestamp: Long,
31
-
onComplete: (AnalysisResult) -> Unit,
85
+
width: Int,
86
+
height: Int,
87
+
onComplete: (AnalysisResult) -> Unit
32
88
) {
89
+
val objectsDetected = objectDetector?.detect(tensorImage)?.map { result->
90
+
AnalysisObject(
91
+
boundingBox = result.boundingBox.let { Rect(
92
+
left = it.left,
93
+
top = it.top,
94
+
right = it.right,
95
+
bottom = it.bottom
96
+
) },
97
+
trackingId = 0,
98
+
labels = result.categories.map { category ->
99
+
com.performancecoachlab.posedetection.recording.Label(
100
+
category.label,
101
+
category.score
102
+
)
103
+
}
104
+
)
105
+
}?: emptyList()
33
106
var skeleton: Skeleton? = null
34
-
var objectsDetected :List<AnalysisObject> = emptyList()
35
-
36
-
val objectDetectionTask = objectDetector?.process(this)?.addOnSuccessListener { detectedObjects ->
37
-
detectedObjects.map { detectedObject ->
38
-
AnalysisObject(
39
-
boundingBox = Rect(
40
-
left = detectedObject.boundingBox.left.toFloat(),
41
-
top = detectedObject.boundingBox.top.toFloat(),
42
-
right = detectedObject.boundingBox.right.toFloat(),
43
-
bottom = detectedObject.boundingBox.bottom.toFloat()
44
-
),
45
-
trackingId = detectedObject.trackingId,
46
-
labels = detectedObject.labels.mapNotNull { if(it.confidence>0)com.performancecoachlab.posedetection.recording.Label(it.text,it.confidence) else null }
47
-
)
48
-
}.also {
49
-
objectsDetected = it
107
+
val poseDetectionTask = mlKitImage?.let {
108
+
poseDetector.process(it).addOnSuccessListener { pose ->
109
+
skeleton = skeleton(pose, timestamp, width, height)
110
+
}.addOnFailureListener { e ->
111
+
println(e)
50
112
}
51
-
}?.addOnFailureListener { e ->
52
-
println(e)
53
113
}
54
-
55
-
val poseDetectionTask = poseDetector.process(this).addOnSuccessListener { pose ->
56
-
skeleton = skeleton(pose, timestamp, width, height)
57
-
}.addOnFailureListener { e ->
58
-
println(e)
59
-
}
60
-
61
-
Tasks.whenAllComplete(listOfNotNull(objectDetectionTask, poseDetectionTask)).addOnCompleteListener {
114
+
Tasks.whenAllComplete(listOfNotNull( poseDetectionTask)).addOnCompleteListener {
62
115
onComplete(
63
116
AnalysisResult(
64
117
skeleton = skeleton,
+27
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.android.kt
+27
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.android.kt
···
1
+
package com.performancecoachlab.posedetection.custom
2
+
3
+
import androidx.compose.runtime.Composable
4
+
import androidx.compose.ui.platform.LocalContext
5
+
6
+
@Composable
7
+
actual fun initialiseObjectModel(modelPath: ModelPath): ObjectModel {
8
+
val options = org.tensorflow.lite.task.vision.detector.ObjectDetector.ObjectDetectorOptions.builder().setMaxResults(10).setScoreThreshold(0.1f).build()
9
+
val detector = org.tensorflow.lite.task.vision.detector.ObjectDetector.createFromFileAndOptions(
10
+
LocalContext.current,
11
+
modelPath.androidModelPath,
12
+
options
13
+
)
14
+
return ObjectModel(detector)
15
+
}
16
+
17
+
actual class ObjectModel{
18
+
private var detector: org.tensorflow.lite.task.vision.detector.ObjectDetector? = null
19
+
20
+
constructor(detector: org.tensorflow.lite.task.vision.detector.ObjectDetector){
21
+
this.detector = detector
22
+
}
23
+
24
+
fun getDetector(): org.tensorflow.lite.task.vision.detector.ObjectDetector? {
25
+
return detector
26
+
}
27
+
}
-23
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/objects/createObjectDetector.kt
-23
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/objects/createObjectDetector.kt
···
1
-
package com.performancecoachlab.posedetection.objects
2
-
3
-
import com.google.mlkit.common.model.LocalModel
4
-
import com.google.mlkit.vision.objects.ObjectDetection
5
-
import com.google.mlkit.vision.objects.ObjectDetector
6
-
import com.google.mlkit.vision.objects.custom.CustomObjectDetectorOptions
7
-
8
-
fun createObjectDetector(objectModel: String?): ObjectDetector? {
9
-
return objectModel?.let {
10
-
val localModel = LocalModel.Builder()
11
-
.setAssetFilePath(it)
12
-
.build()
13
-
val customObjectDetectorOptions =
14
-
CustomObjectDetectorOptions.Builder(localModel)
15
-
.setDetectorMode(CustomObjectDetectorOptions.STREAM_MODE)
16
-
.enableClassification()
17
-
.setClassificationConfidenceThreshold(0.0f)
18
-
.enableMultipleObjects()
19
-
.setMaxPerObjectLabelCount(10)
20
-
.build()
21
-
ObjectDetection.getClient(customObjectDetectorOptions)
22
-
}
23
-
}
+7
-16
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.android.kt
+7
-16
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.android.kt
···
1
1
package com.performancecoachlab.posedetection.recording
2
2
3
3
import android.graphics.Bitmap
4
-
import androidx.compose.ui.geometry.Rect
5
4
import androidx.compose.ui.graphics.ImageBitmap
6
5
import androidx.compose.ui.graphics.asImageBitmap
7
-
import com.google.mlkit.vision.common.InputImage
8
6
import com.google.mlkit.vision.pose.PoseDetection
9
7
import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions
10
8
import com.performancecoachlab.posedetection.camera.drawAnalysisResults
11
9
import com.performancecoachlab.posedetection.camera.drawSkeleton
12
10
import com.performancecoachlab.posedetection.camera.process
13
-
import com.performancecoachlab.posedetection.camera.skeleton
14
-
import com.performancecoachlab.posedetection.custom.CustomObjectModel
15
-
import com.performancecoachlab.posedetection.objects.createObjectDetector
11
+
import com.performancecoachlab.posedetection.custom.CustomObjectDetectorModels
16
12
import com.performancecoachlab.posedetection.skeleton.Skeleton
17
13
import kotlinx.coroutines.suspendCancellableCoroutine
18
14
import kotlin.coroutines.resume
···
32
28
}
33
29
34
30
actual class FrameAnalyser actual constructor() {
35
-
private val modelPath = CustomObjectModel.getInstance().androidModelPath
36
-
private val objectDetector = createObjectDetector(modelPath)
37
31
private val options =
38
32
PoseDetectorOptions.Builder().setDetectorMode(PoseDetectorOptions.STREAM_MODE).build()
39
33
private val poseDetector = PoseDetection.getClient(options)
34
+
private val objDetectors = CustomObjectDetectorModels.getInstance().models
40
35
actual suspend fun analyseFrame(inputFrame: InputFrame): AnalysisResult =
41
36
suspendCancellableCoroutine { continuation ->
42
-
val img = InputImage.fromBitmap(inputFrame.bitmap, 0)
43
-
img.process(
44
-
objectDetector = objectDetector,
45
-
poseDetector = poseDetector,
46
-
timestamp = inputFrame.timestamp,
47
-
onComplete = { result ->
48
-
continuation.resume(result)
49
-
}
50
-
)
37
+
inputFrame.bitmap.process(
38
+
objDetectors.firstOrNull()?.getDetector(), poseDetector, inputFrame.timestamp
39
+
) { result ->
40
+
continuation.resume(result)
41
+
}
51
42
}
52
43
}
+27
-8
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.kt
+27
-8
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.kt
···
1
1
package com.performancecoachlab.posedetection.custom
2
2
3
+
import androidx.compose.runtime.Composable
3
4
import kotlin.concurrent.Volatile
4
5
5
-
class CustomObjectModel private constructor(
6
-
val androidModelPath: String?,
7
-
val iosModelPath: String?
6
+
data class ModelPath(
7
+
val id: Enum<*>,
8
+
val androidModelPath: String? = null,
9
+
val iosModelPath: String? = null,
10
+
)
11
+
12
+
class CustomObjectDetectorModels private constructor(
13
+
val models: List<ObjectModel>,
8
14
) {
9
15
companion object {
10
16
@Volatile
11
-
private var instance = CustomObjectModel(null, null)
17
+
private var instance = CustomObjectDetectorModels(emptyList())
12
18
13
-
fun init(androidModelPath: String?, iosModelPath: String?) {
14
-
instance = CustomObjectModel(androidModelPath, iosModelPath)
19
+
@Composable
20
+
fun init(paths: List<ModelPath>) {
21
+
val models = paths.map { modelPaths ->
22
+
initialiseObjectModel(modelPaths)
23
+
}
24
+
instance = CustomObjectDetectorModels(models = models)
15
25
}
16
26
17
-
fun getInstance(): CustomObjectModel = instance
27
+
fun getInstance(): CustomObjectDetectorModels = instance
18
28
}
19
-
}
29
+
}
30
+
31
+
@Composable
32
+
expect fun initialiseObjectModel(modelPath: ModelPath): ObjectModel
33
+
34
+
@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
35
+
expect class ObjectModel {
36
+
37
+
}
38
+
+62
-34
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
+62
-34
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
···
1
1
package com.performancecoachlab.posedetection.camera
2
2
3
+
import androidx.compose.ui.geometry.Offset
4
+
import androidx.compose.ui.geometry.Rect
3
5
import androidx.compose.ui.geometry.Size
4
6
import androidx.compose.ui.graphics.Canvas
5
7
import androidx.compose.ui.graphics.ImageBitmap
···
8
10
import androidx.compose.ui.graphics.toComposeImageBitmap
9
11
import androidx.compose.ui.unit.Density
10
12
import androidx.compose.ui.unit.LayoutDirection
11
-
import com.performancecoachlab.posedetection.custom.CustomObjectModel
13
+
import com.performancecoachlab.posedetection.custom.CustomObjectDetectorModels
12
14
import com.performancecoachlab.posedetection.custom.CustomObjectRespository
13
-
import com.performancecoachlab.posedetection.recording.createObjectDetector
15
+
import com.performancecoachlab.posedetection.recording.AnalysisObject
16
+
import com.performancecoachlab.posedetection.recording.AnalysisResult
14
17
import com.performancecoachlab.posedetection.skeleton.Skeleton
15
18
import com.performancecoachlab.posedetection.skeleton.SkeletonRepository
16
19
import kotlinx.cinterop.ExperimentalForeignApi
···
435
438
onError?.invoke(CameraException.ConfigurationError(e.message ?: "Unknown error"))
436
439
}
437
440
}
438
-
private val modelPath = CustomObjectModel.getInstance().iosModelPath
439
-
private val objectDetector = createObjectDetector(modelPath)
441
+
442
+
private val objectDetector = CustomObjectDetectorModels.getInstance().models.firstOrNull()?.getModel()
440
443
private val frameProcessor = FrameProcessor(objectDetector)
441
444
442
445
@OptIn(ExperimentalForeignApi::class, NativeRuntimeApi::class)
···
452
455
).also { processingQueue ->
453
456
dispatch_async(processingQueue) {
454
457
try {
458
+
var detectedSkeleton: Skeleton? = null
459
+
var detectedObjects: List<AnalysisObject> = emptyList()
455
460
frameProcessor.analyseBufferForAll(
456
461
CMSampleBufferGetImageBuffer(didOutputSampleBuffer), timestamp,
457
462
onSkeletonProcessed = { skeleton ->
458
-
cameraPreviewLayer?.also { preview ->
459
-
skeleton?.let {
460
-
mapSkeletonToPreview(
461
-
skeleton = it,
462
-
previewLayer = preview,
463
-
isUsingFrontCamera = isUsingFrontCamera,
464
-
width = 480f,
465
-
height = 360f
466
-
)
467
-
}?.also { updatedSkeleton ->
468
-
skeletonRepository?.updateSkeleton(
469
-
updatedSkeleton
470
-
)
471
-
preview.bounds.useContents {
472
-
Pair(
473
-
size.width.toInt(),
474
-
size.height.toInt()
475
-
)
476
-
}.also { bo ->
477
-
ImageBitmap(
478
-
bo.first, bo.second
479
-
).drawSkeleton(inskeleton = updatedSkeleton)
480
-
.also { drawn ->
481
-
frameListener?.updateFrame(
482
-
drawn
483
-
)
484
-
}
485
-
}
486
-
}
463
+
skeleton?.also {
464
+
skeletonRepository?.updateSkeleton(
465
+
it
466
+
)
487
467
}
468
+
detectedSkeleton = skeleton
488
469
},
489
470
onObjectsProcessed = { objects ->
471
+
detectedObjects = objects
490
472
customObjectRepository?.updateCustomObject(objects)
491
473
}
492
474
)
475
+
cameraPreviewLayer?.also { preview ->
476
+
val previewSkeleton = detectedSkeleton?.let {
477
+
mapSkeletonToPreview(
478
+
skeleton = it,
479
+
previewLayer = preview,
480
+
width = 480f,
481
+
height = 360f
482
+
)
483
+
}
484
+
val previewObjects = detectedObjects.map {
485
+
it.copy(boundingBox = mapBoxToPreview(it.boundingBox,preview,
486
+
width = 480f,
487
+
height = 360f))
488
+
}
489
+
preview.bounds.useContents {
490
+
Pair(
491
+
size.width.toInt(),
492
+
size.height.toInt()
493
+
)
494
+
}.also { bo ->
495
+
ImageBitmap(
496
+
bo.first, bo.second
497
+
).drawAnalysisResults(analysisResults = AnalysisResult(skeleton = previewSkeleton, objects = previewObjects))
498
+
.also { drawn ->
499
+
frameListener?.updateFrame(
500
+
drawn
501
+
)
502
+
}
503
+
}
504
+
}
493
505
} catch (e: Exception) {
494
506
println(e.message ?: "Unknown error in frame processing")
495
507
}
···
531
543
}
532
544
533
545
@OptIn(ExperimentalForeignApi::class)
546
+
fun mapBoxToPreview(
547
+
box: Rect,
548
+
previewLayer: AVCaptureVideoPreviewLayer,
549
+
width: Float,
550
+
height: Float,
551
+
): Rect {
552
+
fun mapPoint(point: Offset): Offset {
553
+
val normalizedPoint = CGPointMake(point.x.toDouble()/width, point.y.toDouble()/height)
554
+
val screenPoint = previewLayer.pointForCaptureDevicePointOfInterest(normalizedPoint)
555
+
return Offset(screenPoint.useContents { x.toFloat() }, screenPoint.useContents { y.toFloat() })
556
+
}
557
+
val topLeft = mapPoint(box.topLeft)
558
+
val bottomRight = mapPoint(box.bottomRight)
559
+
return Rect(topLeft = topLeft, bottomRight = bottomRight)
560
+
}
561
+
562
+
@OptIn(ExperimentalForeignApi::class)
534
563
fun mapSkeletonToPreview(
535
564
skeleton: Skeleton,
536
565
previewLayer: AVCaptureVideoPreviewLayer,
537
-
isUsingFrontCamera: Boolean,
538
566
width: Float,
539
567
height: Float
540
568
): Skeleton {
-273
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
-273
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
···
148
148
}
149
149
}
150
150
151
-
fun detectObjects(pixelBuffer: CVPixelBufferRef) {
152
-
val handler = VNImageRequestHandler(pixelBuffer, mapOf<Any?, Any?>())
153
-
try {
154
-
handler.performRequests(requests, null)
155
-
} catch (e: Throwable) {
156
-
println("Detection error: ${e.message}")
157
-
}
158
-
}
159
-
160
151
private fun processResults(results: List<VNRecognizedObjectObservation>) {
161
152
val detectedObjects = results.map { observation ->
162
153
val label = observation.labels.firstOrNull()?.toString() ?: "Unknown"
···
165
156
DetectedObject(
166
157
label = label, confidence = confidence, boundingBox = boundingBox
167
158
)
168
-
}
169
-
}
170
-
171
-
@OptIn(BetaInteropApi::class)
172
-
fun analyseBuffer(
173
-
buffer: CVImageBufferRef?, timestamp: Long, onProcessed: (Skeleton?) -> Unit
174
-
) {
175
-
if (buffer == null) {
176
-
onProcessed(null)
177
-
return
178
-
}
179
-
val width = 480uL
180
-
val height = 360uL
181
-
memScoped {
182
-
val errorPtr = alloc<ObjCObjectVar<NSError?>>()
183
-
val options = mapOf<Any?, Any?>(
184
-
"orientation" to AVCaptureVideoOrientationLandscapeRight
185
-
)
186
-
val requestHandler = VNImageRequestHandler(buffer, options)
187
-
val request = VNDetectHumanBodyPoseRequest { request, error ->
188
-
if (error != null) {
189
-
onProcessed(null)
190
-
} else {
191
-
request?.also { vnRequest ->
192
-
val recognizedPoints = bodyPoseHandler(vnRequest)
193
-
regionOfInterest = calculateRegionOfInterest(recognizedPoints)
194
-
val updatedSkeleton = Skeleton(
195
-
timestamp = timestamp,
196
-
leftShoulder = recognizedPoints?.get(
197
-
VNHumanBodyPoseObservationJointNameLeftShoulder
198
-
)?.location?.toSkeletonPoint(width, height),
199
-
rightShoulder = recognizedPoints?.get(
200
-
VNHumanBodyPoseObservationJointNameRightShoulder
201
-
)?.location?.toSkeletonPoint(width, height),
202
-
leftElbow = recognizedPoints?.get(
203
-
VNHumanBodyPoseObservationJointNameLeftElbow
204
-
)?.location?.toSkeletonPoint(width, height),
205
-
rightElbow = recognizedPoints?.get(
206
-
VNHumanBodyPoseObservationJointNameRightElbow
207
-
)?.location?.toSkeletonPoint(width, height),
208
-
leftWrist = recognizedPoints?.get(
209
-
VNHumanBodyPoseObservationJointNameLeftWrist
210
-
)?.location?.toSkeletonPoint(width, height),
211
-
rightWrist = recognizedPoints?.get(
212
-
VNHumanBodyPoseObservationJointNameRightWrist
213
-
)?.location?.toSkeletonPoint(width, height),
214
-
leftHip = recognizedPoints?.get(
215
-
VNHumanBodyPoseObservationJointNameLeftHip
216
-
)?.location?.toSkeletonPoint(width, height),
217
-
rightHip = recognizedPoints?.get(
218
-
VNHumanBodyPoseObservationJointNameRightHip
219
-
)?.location?.toSkeletonPoint(width, height),
220
-
leftKnee = recognizedPoints?.get(
221
-
VNHumanBodyPoseObservationJointNameLeftKnee
222
-
)?.location?.toSkeletonPoint(width, height),
223
-
rightKnee = recognizedPoints?.get(
224
-
VNHumanBodyPoseObservationJointNameRightKnee
225
-
)?.location?.toSkeletonPoint(width, height),
226
-
leftAnkle = recognizedPoints?.get(
227
-
VNHumanBodyPoseObservationJointNameLeftAnkle
228
-
)?.location?.toSkeletonPoint(width, height),
229
-
rightAnkle = recognizedPoints?.get(
230
-
VNHumanBodyPoseObservationJointNameRightAnkle
231
-
)?.location?.toSkeletonPoint(width, height),
232
-
height = height.toFloat(),
233
-
width = width.toFloat()
234
-
)
235
-
onProcessed(skelBuffer.smooth(updatedSkeleton))
236
-
}
237
-
}
238
-
}
239
-
request.regionOfInterest = regionOfInterest
240
-
try {
241
-
val result = runCatching {
242
-
request(requestHandler, request, errorPtr)
243
-
}
244
-
245
-
if (result.isFailure) {
246
-
println("Exception during performRequests: ${result.exceptionOrNull()?.message}")
247
-
onProcessed(null)
248
-
return@memScoped
249
-
}
250
-
251
-
if (errorPtr.value != null) {
252
-
println("Error performing request: ${errorPtr.value}")
253
-
onProcessed(null)
254
-
}
255
-
} catch (e: Throwable) {
256
-
println("Unable to perform the request: ${e.message}")
257
-
onProcessed(null)
258
-
}
259
-
}
260
-
}
261
-
262
-
@OptIn(BetaInteropApi::class)
263
-
fun analyseFrame(cgImage: CGImageRef?, timestamp: Long, onProccessed: (Skeleton?) -> Unit) {
264
-
autoreleasepool {
265
-
if (cgImage == null) {
266
-
onProccessed(null)
267
-
return
268
-
}
269
-
val width = CGImageGetWidth(cgImage)
270
-
val height = CGImageGetHeight(cgImage)
271
-
print("width: $width, height: $height")
272
-
if (width.toUInt() == 0u || height.toUInt() == 0u) {
273
-
onProccessed(null)
274
-
return
275
-
}
276
-
memScoped {
277
-
val errorPtr = alloc<ObjCObjectVar<NSError?>>()
278
-
val requestHandler = VNImageRequestHandler(cgImage, mapOf<Any?, Any?>())
279
-
val request = VNDetectHumanBodyPoseRequest { request, error ->
280
-
if (error != null) {
281
-
onProccessed(null)
282
-
} else {
283
-
request?.also { vnRequest ->
284
-
val recognizedPoints = bodyPoseHandler(vnRequest)
285
-
regionOfInterest = calculateRegionOfInterest(recognizedPoints)
286
-
val updatedSkeleton = Skeleton(
287
-
timestamp = timestamp,
288
-
leftShoulder = recognizedPoints?.get(
289
-
VNHumanBodyPoseObservationJointNameLeftShoulder
290
-
)?.location?.toSkeletonPoint(width, height),
291
-
rightShoulder = recognizedPoints?.get(
292
-
VNHumanBodyPoseObservationJointNameRightShoulder
293
-
)?.location?.toSkeletonPoint(width, height),
294
-
leftElbow = recognizedPoints?.get(
295
-
VNHumanBodyPoseObservationJointNameLeftElbow
296
-
)?.location?.toSkeletonPoint(width, height),
297
-
rightElbow = recognizedPoints?.get(
298
-
VNHumanBodyPoseObservationJointNameRightElbow
299
-
)?.location?.toSkeletonPoint(width, height),
300
-
leftWrist = recognizedPoints?.get(
301
-
VNHumanBodyPoseObservationJointNameLeftWrist
302
-
)?.location?.toSkeletonPoint(width, height),
303
-
rightWrist = recognizedPoints?.get(
304
-
VNHumanBodyPoseObservationJointNameRightWrist
305
-
)?.location?.toSkeletonPoint(width, height),
306
-
leftHip = recognizedPoints?.get(
307
-
VNHumanBodyPoseObservationJointNameLeftHip
308
-
)?.location?.toSkeletonPoint(width, height),
309
-
rightHip = recognizedPoints?.get(
310
-
VNHumanBodyPoseObservationJointNameRightHip
311
-
)?.location?.toSkeletonPoint(width, height),
312
-
leftKnee = recognizedPoints?.get(
313
-
VNHumanBodyPoseObservationJointNameLeftKnee
314
-
)?.location?.toSkeletonPoint(width, height),
315
-
rightKnee = recognizedPoints?.get(
316
-
VNHumanBodyPoseObservationJointNameRightKnee
317
-
)?.location?.toSkeletonPoint(width, height),
318
-
leftAnkle = recognizedPoints?.get(
319
-
VNHumanBodyPoseObservationJointNameLeftAnkle
320
-
)?.location?.toSkeletonPoint(width, height),
321
-
rightAnkle = recognizedPoints?.get(
322
-
VNHumanBodyPoseObservationJointNameRightAnkle
323
-
)?.location?.toSkeletonPoint(width, height),
324
-
height = height.toFloat(),
325
-
width = width.toFloat()
326
-
)
327
-
onProccessed(skelBuffer.smooth(updatedSkeleton))
328
-
}
329
-
}
330
-
}
331
-
request.regionOfInterest = regionOfInterest
332
-
try {
333
-
val result = runCatching {
334
-
request(requestHandler, request, errorPtr)
335
-
}
336
-
337
-
if (result.isFailure) {
338
-
println("Exception during performRequests: ${result.exceptionOrNull()?.message}")
339
-
onProccessed(null)
340
-
return@memScoped
341
-
}
342
-
343
-
if (errorPtr.value != null) {
344
-
println("Error performing request: ${errorPtr.value}")
345
-
onProccessed(null)
346
-
}
347
-
} catch (e: Throwable) {
348
-
println("Unable to perform the request: ${e.message}")
349
-
onProccessed(null)
350
-
}
351
-
}
352
-
}
353
-
}
354
-
355
-
@OptIn(BetaInteropApi::class, ExperimentalNativeApi::class)
356
-
fun analyseFrameForObjects(
357
-
cgImage: CGImageRef?, timestamp: Long, onProccessed: (List<AnalysisObject>) -> Unit
358
-
) {
359
-
autoreleasepool {
360
-
if (cgImage == null) {
361
-
onProccessed(emptyList())
362
-
return
363
-
}
364
-
val width = CGImageGetWidth(cgImage)
365
-
val height = CGImageGetHeight(cgImage)
366
-
if (width.toUInt() == 0u || height.toUInt() == 0u) {
367
-
onProccessed(emptyList())
368
-
return
369
-
}
370
-
memScoped {
371
-
val errorPtr = alloc<ObjCObjectVar<NSError?>>()
372
-
if (modelObj == null) {
373
-
onProccessed(emptyList())
374
-
return@memScoped
375
-
}
376
-
try {
377
-
val request = VNCoreMLRequest(modelObj) { request, error ->
378
-
if (error != null) {
379
-
onProccessed(emptyList())
380
-
return@VNCoreMLRequest
381
-
}
382
-
val results = request?.results as? List<*> ?: emptyList<Any>()
383
-
val recognized = results.filterIsInstance<VNRecognizedObjectObservation>()
384
-
val analysisObjects = recognized.map { observation ->
385
-
val confidence = observation.confidence
386
-
val boundingBox = observation.boundingBox.useContents {
387
-
val w = width.toFloat()
388
-
val h = height.toFloat()
389
-
val left = origin.x * w
390
-
val top = (1.0 - origin.y) * h
391
-
val right = (origin.x + size.width) * w
392
-
val bottom = (1.0 - (origin.y + size.height)) * h
393
-
Rect(
394
-
left = left.toFloat(),
395
-
top = top.toFloat(),
396
-
right = right.toFloat(),
397
-
bottom = bottom.toFloat()
398
-
)
399
-
}
400
-
val labels = observation.labels.mapNotNull {
401
-
(it as VNClassificationObservation).let { ca ->
402
-
if (ca.confidence > 0.0) Label(ca.identifier,ca.confidence) else null
403
-
}
404
-
}
405
-
AnalysisObject(
406
-
trackingId = observation.identityHashCode(),
407
-
labels = labels,
408
-
boundingBox = boundingBox,
409
-
)
410
-
}
411
-
onProccessed(analysisObjects)
412
-
}
413
-
val handler = VNImageRequestHandler(cgImage, mapOf<Any?, Any?>())
414
-
handler.performRequests(listOf(request), errorPtr.ptr)
415
-
if (errorPtr.value != null) {
416
-
println("Error performing object detection request: ${errorPtr.value}")
417
-
onProccessed(emptyList())
418
-
}
419
-
} catch (e: Throwable) {
420
-
println("Unable to perform the object detection request: ${e.message}")
421
-
onProccessed(emptyList())
422
-
}
423
-
}
424
159
}
425
160
}
426
161
···
726
461
val confidence: Float,
727
462
val boundingBox: CValue<CGRect>
728
463
)
729
-
730
-
@OptIn(ExperimentalForeignApi::class)
731
-
@Throws(Throwable::class)
732
-
fun request(
733
-
requestHandler: VNImageRequestHandler, request: VNRequest, errorPtr: ObjCObjectVar<NSError?>
734
-
) {
735
-
requestHandler.performRequests(listOf(request), errorPtr.ptr)
736
-
}
+42
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.ios.kt
+42
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.ios.kt
···
1
+
package com.performancecoachlab.posedetection.custom
2
+
3
+
import androidx.compose.runtime.Composable
4
+
import kotlinx.cinterop.ExperimentalForeignApi
5
+
import platform.CoreML.MLModel
6
+
import platform.Foundation.NSBundle
7
+
import platform.Foundation.NSURL
8
+
import platform.Vision.VNCoreMLModel
9
+
10
+
@Composable
11
+
actual fun initialiseObjectModel(modelPath: ModelPath): ObjectModel {
12
+
val model = createObjectDetector(modelPath.iosModelPath)
13
+
return ObjectModel(model)
14
+
}
15
+
16
+
@OptIn(ExperimentalForeignApi::class)
17
+
fun createObjectDetector(model: String?): VNCoreMLModel? {
18
+
if(model == null) {
19
+
println("Model path is null")
20
+
return null
21
+
}
22
+
println("Model input: $model")
23
+
val path = NSBundle.mainBundle.pathForResource(model, "mlmodelc")
24
+
println("Model path: $path")
25
+
val url = path?.let { NSURL.fileURLWithPath(it) }
26
+
println("Model URL: $url")
27
+
val modelCont = url?.let {MLModel.modelWithContentsOfURL(it, null)}
28
+
println("Model content: $modelCont")
29
+
val modelObj = modelCont?.let {VNCoreMLModel.modelForMLModel(it,null)}
30
+
println("Model: $modelObj")
31
+
return modelObj
32
+
}
33
+
34
+
actual class ObjectModel{
35
+
private var model: VNCoreMLModel? = null
36
+
constructor(model: VNCoreMLModel?){
37
+
this.model = model
38
+
}
39
+
fun getModel(): VNCoreMLModel? {
40
+
return model
41
+
}
42
+
}
+2
-21
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.ios.kt
+2
-21
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.ios.kt
···
5
5
import com.performancecoachlab.posedetection.camera.drawAnalysisResults
6
6
import com.performancecoachlab.posedetection.camera.drawSkeleton
7
7
import com.performancecoachlab.posedetection.camera.toImageBitmap
8
-
import com.performancecoachlab.posedetection.custom.CustomObjectModel
8
+
import com.performancecoachlab.posedetection.custom.CustomObjectDetectorModels
9
9
import com.performancecoachlab.posedetection.skeleton.Skeleton
10
10
import kotlinx.cinterop.ExperimentalForeignApi
11
11
import kotlinx.coroutines.suspendCancellableCoroutine
···
34
34
}
35
35
}
36
36
37
-
@OptIn(ExperimentalForeignApi::class)
38
-
fun createObjectDetector(model: String?): VNCoreMLModel? {
39
-
if(model == null) {
40
-
println("Model path is null")
41
-
return null
42
-
}
43
-
println("Model input: $model")
44
-
val path = NSBundle.mainBundle.pathForResource(model, "mlmodelc")
45
-
println("Model path: $path")
46
-
val url = path?.let { NSURL.fileURLWithPath(it) }
47
-
println("Model URL: $url")
48
-
val modelCont = url?.let {MLModel.modelWithContentsOfURL(it, null)}
49
-
println("Model content: $modelCont")
50
-
val modelObj = modelCont?.let {VNCoreMLModel.modelForMLModel(it,null)}
51
-
println("Model: $modelObj")
52
-
return modelObj
53
-
}
54
-
55
37
actual class FrameAnalyser actual constructor() {
56
-
private val modelPath = CustomObjectModel.getInstance().iosModelPath
57
-
private val modelObj:VNCoreMLModel? = createObjectDetector(modelPath)
38
+
private val modelObj = CustomObjectDetectorModels.getInstance().models.firstOrNull()?.getModel()
58
39
private val frameProcessor = FrameProcessor(modelObj)
59
40
60
41
@OptIn(ExperimentalForeignApi::class)
+23
-6
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
+23
-6
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
···
38
38
import com.nate.posedetection.theme.AppTheme
39
39
import com.performancecoachlab.posedetection.camera.CameraView
40
40
import com.performancecoachlab.posedetection.camera.drawSkeleton
41
-
import com.performancecoachlab.posedetection.custom.CustomObjectModel
41
+
import com.performancecoachlab.posedetection.custom.CustomObjectDetectorModels
42
42
import com.performancecoachlab.posedetection.custom.CustomObjectRespository
43
+
import com.performancecoachlab.posedetection.custom.ModelPath
43
44
import com.performancecoachlab.posedetection.encoding.VideoBuilder
44
45
import com.performancecoachlab.posedetection.encoding.createVideoBuilder
45
46
import com.performancecoachlab.posedetection.permissions.PermissionProvider
···
61
62
import kotlinx.coroutines.launch
62
63
import kotlin.math.roundToLong
63
64
65
+
66
+
enum class CustomModels{
67
+
DEFAULT
68
+
}
69
+
64
70
@Composable
65
71
internal fun App() = AppTheme {
66
72
var selectedTabIndex by remember { mutableStateOf(0) }
67
73
val tabs = listOf("Camera Feed", "Recorded Video")
68
-
CustomObjectModel.init(
69
-
androidModelPath = "4.tflite",
70
-
iosModelPath = "YOLOv3FP16"
74
+
75
+
CustomObjectDetectorModels.init(
76
+
listOf(ModelPath(
77
+
CustomModels.DEFAULT,"latest.tflite", "YOLOv3FP16"
78
+
))
71
79
)
80
+
72
81
73
82
Column {
74
83
TabRow(selectedTabIndex = selectedTabIndex) {
···
188
197
val extractedFrame = extractFrame(url, frame)
189
198
extractedFrame?.let { frame ->
190
199
frameAnalyser.analyseFrame(frame).let { analysisResults ->
200
+
analysisResults.objects.let {
201
+
if (it.isNotEmpty()) {
202
+
it.forEach { obj ->
203
+
println("Detected Objects: ${obj.labels}")
204
+
}
205
+
}
206
+
}
191
207
frame.drawAnalysisResults(analysisResults).also {
192
208
if (!isRecording) {
193
209
size = Pair(it.width, it.height)
···
320
336
customObjectRepository = customObjectRespository,
321
337
drawSkeleton = true,
322
338
modifier = Modifier.weight(1f),
323
-
frontCamera = true,
339
+
frontCamera = false,
324
340
isRecording = isRecording,
325
341
onRecordToggled = { isRecording = it },
326
342
onVideoSaved = { path = it },
···
352
368
if (it.isNotEmpty()) {
353
369
it.forEach { obj ->
354
370
val l = "${obj.labels.maxByOrNull { it.confidence }?.text}"
371
+
println("Detected Objects: ${obj.labels}")
355
372
if(!allDetected.contains(l)){
356
373
allDetected.add(l)
357
-
println(allDetected)
374
+
println("Detected Objects: $allDetected")
358
375
}
359
376
}
360
377
}