This repository has no description
feat: specify object for each usecase
author
nathan holland
date
(Jul 27, 2025, 11:32 PM +0100)
commit
64b4b87a
64b4b87a3ea28137c7a0204c8115c404595c3acd
parent
1c61763f
1c61763f5f29f55ecf22d5612bdcaea10283abc5
+84
-79
+11
-4
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
+11
-4
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
···
47
47
import androidx.compose.runtime.rememberCoroutineScope
48
48
import androidx.compose.ui.geometry.Rect
49
49
import com.google.mlkit.vision.pose.PoseDetector
50
-
import com.performancecoachlab.posedetection.custom.CustomObjectDetectorModels
51
50
import com.performancecoachlab.posedetection.custom.CustomObjectRespository
51
+
import com.performancecoachlab.posedetection.custom.ObjectModel
52
52
import com.performancecoachlab.posedetection.recording.AnalysisObject
53
53
import com.performancecoachlab.posedetection.recording.AnalysisResult
54
54
import kotlinx.coroutines.launch
···
61
61
skeletonRepository: SkeletonRepository,
62
62
customObjectRepository: CustomObjectRespository,
63
63
drawSkeleton: Boolean,
64
+
objectModel: ObjectModel?,
64
65
drawObjects: Boolean,
65
66
modifier: Modifier,
66
67
frontCamera: Boolean,
···
77
78
val lifecycleOwner: LifecycleOwner = LocalLifecycleOwner.current
78
79
val previewView: PreviewView = remember { PreviewView(context) }
79
80
val executor = remember { Executors.newSingleThreadExecutor() }
80
-
val objDetector = CustomObjectDetectorModels.getInstance().model
81
81
val scope = rememberCoroutineScope()
82
82
var firstFrameTimestamp: Long? = null
83
-
var focus by remember { mutableStateOf<Rect?>(focusArea) }
83
+
var focus by remember { mutableStateOf(focusArea) }
84
+
var objectDetector by remember { mutableStateOf(objectModel) }
84
85
85
86
// Update focus when focusArea changes
86
87
LaunchedEffect(focusArea) {
87
88
focus = focusArea
89
+
}
90
+
91
+
// Update object model when it changes
92
+
LaunchedEffect(objectModel) {
93
+
objectDetector = objectModel
88
94
}
89
95
90
96
// Video recording state
···
144
150
analysis.setAnalyzer(executor) { imageProxy ->
145
151
val timestamp = System.currentTimeMillis()
146
152
val area = focus
153
+
val currentObjectDetector = objectDetector?.getDetector()
147
154
imageProxy.process(
148
-
objDetector?.getDetector(), poseDetector, timestamp, area
155
+
currentObjectDetector, poseDetector, timestamp, area
149
156
){
150
157
customObjectRepository.updateCustomObject(it.objects)
151
158
it.skeleton?.let { skel ->
+3
-3
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.android.kt
+3
-3
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.android.kt
···
9
9
import com.performancecoachlab.posedetection.camera.drawAnalysisResults
10
10
import com.performancecoachlab.posedetection.camera.drawSkeleton
11
11
import com.performancecoachlab.posedetection.camera.process
12
-
import com.performancecoachlab.posedetection.custom.CustomObjectDetectorModels
12
+
import com.performancecoachlab.posedetection.custom.ObjectModel
13
13
import com.performancecoachlab.posedetection.skeleton.Skeleton
14
14
import kotlinx.coroutines.suspendCancellableCoroutine
15
15
import kotlin.coroutines.resume
···
28
28
}
29
29
}
30
30
31
-
actual class FrameAnalyser actual constructor() {
31
+
actual class FrameAnalyser actual constructor(val model: ObjectModel?) {
32
32
private val options =
33
33
PoseDetectorOptions.Builder().setDetectorMode(PoseDetectorOptions.STREAM_MODE).build()
34
34
private val poseDetector = PoseDetection.getClient(options)
35
-
private val objDetector = CustomObjectDetectorModels.getInstance().model?.getDetector()
35
+
private val objDetector = model?.getDetector()
36
36
actual suspend fun analyseFrame(inputFrame: InputFrame, focusArea: Rect?): AnalysisResult =
37
37
suspendCancellableCoroutine { continuation ->
38
38
inputFrame.bitmap.process(
+2
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.kt
+2
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.kt
···
5
5
import androidx.compose.ui.Modifier
6
6
import androidx.compose.ui.geometry.Rect
7
7
import com.performancecoachlab.posedetection.custom.CustomObjectRespository
8
+
import com.performancecoachlab.posedetection.custom.ObjectModel
8
9
import com.performancecoachlab.posedetection.skeleton.SkeletonRepository
9
10
10
11
@Composable
···
12
13
skeletonRepository: SkeletonRepository,
13
14
customObjectRepository: CustomObjectRespository,
14
15
drawSkeleton: Boolean = true,
16
+
objectModel: ObjectModel? = null,
15
17
drawObjects: Boolean = true,
16
18
modifier: Modifier = Modifier.fillMaxSize(),
17
19
frontCamera: Boolean = true,
-18
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.kt
-18
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.kt
···
8
8
val iosModelPath: String? = null,
9
9
)
10
10
11
-
class CustomObjectDetectorModels private constructor(
12
-
val model: ObjectModel?
13
-
) {
14
-
companion object {
15
-
@Volatile
16
-
private var instance = CustomObjectDetectorModels(null)
17
-
18
-
@Composable
19
-
fun init(path: ModelPath) {
20
-
val model = initialiseObjectModel(path)
21
-
instance = CustomObjectDetectorModels(model = model)
22
-
}
23
-
24
-
fun getInstance(): CustomObjectDetectorModels = instance
25
-
}
26
-
}
27
-
28
11
@Composable
29
12
expect fun initialiseObjectModel(modelPath: ModelPath): ObjectModel
30
13
31
14
@Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING")
32
15
expect class ObjectModel {
33
-
34
16
}
35
17
+2
-1
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.kt
+2
-1
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.kt
···
2
2
3
3
import androidx.compose.ui.geometry.Rect
4
4
import androidx.compose.ui.graphics.ImageBitmap
5
+
import com.performancecoachlab.posedetection.custom.ObjectModel
5
6
import com.performancecoachlab.posedetection.skeleton.Skeleton
6
7
7
8
expect class InputFrame {
···
12
13
val timestamp: Long
13
14
}
14
15
15
-
expect class FrameAnalyser() {
16
+
expect class FrameAnalyser(model: ObjectModel?) {
16
17
suspend fun analyseFrame(inputFrame: InputFrame, focusArea: Rect? = null): AnalysisResult
17
18
}
18
19
+10
-17
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
+10
-17
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
···
10
10
import androidx.compose.ui.graphics.toComposeImageBitmap
11
11
import androidx.compose.ui.unit.Density
12
12
import androidx.compose.ui.unit.LayoutDirection
13
-
import com.performancecoachlab.posedetection.custom.CustomObjectDetectorModels
14
13
import com.performancecoachlab.posedetection.custom.CustomObjectRespository
14
+
import com.performancecoachlab.posedetection.custom.ObjectModel
15
15
import com.performancecoachlab.posedetection.recording.AnalysisObject
16
16
import com.performancecoachlab.posedetection.recording.AnalysisResult
17
17
import com.performancecoachlab.posedetection.skeleton.Skeleton
···
227
227
fun setFocusArea(focusArea: Rect?) {
228
228
cameraController.setFocusArea(focusArea)
229
229
}
230
+
231
+
fun setObjectModel(objectModel: ObjectModel?){
232
+
cameraController.setObjectModel(objectModel)
233
+
}
230
234
}
231
235
232
236
class CameraController : NSObject(), AVCaptureVideoDataOutputSampleBufferDelegateProtocol,
···
293
297
frameProcessor.setFocusArea(focusArea)
294
298
}
295
299
300
+
fun setObjectModel(objectModel: ObjectModel?){
301
+
frameProcessor.setObjectModel(objectModel)
302
+
}
303
+
296
304
fun setupSession() {
297
305
try {
298
306
captureSession = AVCaptureSession()
···
460
468
}
461
469
}
462
470
463
-
private val objectDetector = CustomObjectDetectorModels.getInstance().model?.getModel()
464
-
private val frameProcessor = FrameProcessor(objectDetector)
471
+
private val frameProcessor = FrameProcessor(null)
465
472
466
473
@OptIn(ExperimentalForeignApi::class, NativeRuntimeApi::class)
467
474
override fun captureOutput(
···
481
488
frameProcessor.analyseBufferForAll(
482
489
CMSampleBufferGetImageBuffer(didOutputSampleBuffer),
483
490
timestamp,
484
-
mapPoint = { point: Skeleton.SkeletonCoordinate ->
485
-
val normalizedPoint = CGPointMake(
486
-
point.x.toDouble() / 480f, point.y.toDouble() / 360f
487
-
)
488
-
cameraPreviewLayer?.let { preview ->
489
-
val screenPoint =
490
-
preview.pointForCaptureDevicePointOfInterest(
491
-
normalizedPoint
492
-
)
493
-
Skeleton.SkeletonCoordinate(
494
-
screenPoint.useContents { x.toFloat() },
495
-
screenPoint.useContents { y.toFloat() })
496
-
}?:point
497
-
},
498
491
preview = cameraPreviewLayer,
499
492
onSkeletonProcessed = { skeleton ->
500
493
skeleton?.also {
+13
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt
+13
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt
···
9
9
import androidx.compose.runtime.getValue
10
10
import androidx.compose.runtime.mutableStateOf
11
11
import androidx.compose.runtime.remember
12
+
import androidx.compose.runtime.rememberCoroutineScope
12
13
import androidx.compose.runtime.setValue
13
14
import androidx.compose.ui.Modifier
14
15
import androidx.compose.ui.geometry.Rect
15
16
import androidx.compose.ui.layout.ContentScale
16
17
import com.performancecoachlab.posedetection.custom.CustomObjectRespository
18
+
import com.performancecoachlab.posedetection.custom.ObjectModel
17
19
import com.performancecoachlab.posedetection.skeleton.SkeletonRepository
18
20
import kotlinx.cinterop.BetaInteropApi
19
21
import kotlinx.cinterop.autoreleasepool
22
+
import kotlinx.coroutines.delay
20
23
import kotlin.native.runtime.NativeRuntimeApi
21
24
22
25
@OptIn(NativeRuntimeApi::class, BetaInteropApi::class)
···
25
28
skeletonRepository: SkeletonRepository,
26
29
customObjectRepository:CustomObjectRespository,
27
30
drawSkeleton: Boolean,
31
+
objectModel: ObjectModel?,
28
32
drawObjects: Boolean,
29
33
modifier: Modifier,
30
34
frontCamera: Boolean,
···
54
58
}
55
59
LaunchedEffect(focusArea){
56
60
cameraEngine.value?.setFocusArea(focusArea)
61
+
}
62
+
LaunchedEffect(objectModel) {
63
+
cameraEngine.value?.setObjectModel(objectModel)
64
+
}
65
+
LaunchedEffect(Unit){
66
+
delay(1000L)
67
+
cameraEngine.value?.setObjectModel(objectModel)
68
+
cameraEngine.value?.setFocusArea(focusArea)
69
+
println("model set")
57
70
}
58
71
Box(modifier = Modifier.fillMaxSize()) {
59
72
CameraPreview(
+18
-13
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
+18
-13
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
···
2
2
3
3
import androidx.compose.ui.geometry.Offset
4
4
import androidx.compose.ui.geometry.Rect
5
+
import com.performancecoachlab.posedetection.custom.ObjectModel
5
6
import com.performancecoachlab.posedetection.recording.AnalysisObject
6
7
import com.performancecoachlab.posedetection.recording.Label
7
8
import com.performancecoachlab.posedetection.skeleton.Skeleton
···
127
128
}
128
129
129
130
@OptIn(ExperimentalForeignApi::class)
130
-
class FrameProcessor(val modelObj: VNCoreMLModel?) {
131
+
class FrameProcessor(var modelObj: VNCoreMLModel?) {
131
132
private val skelBuffer = SkelBuffer(maxSize = 10)
132
133
private var regionOfInterest = CGRectMake(0.0, 0.0, 1.0, 1.0)
133
134
private var requests = mutableListOf<VNRequest>()
134
135
val objectRecognition = setUpRecognition()
135
136
private var focusArea: Rect? = null
137
+
private var path = ""
136
138
137
139
fun setFocusArea(focusArea: Rect?) {
138
140
this.focusArea = focusArea
141
+
}
142
+
143
+
fun setObjectModel(objectModel: ObjectModel?){
144
+
modelObj = objectModel?.getModel()
145
+
setUpRecognition()
139
146
}
140
147
141
148
private fun setUpRecognition() {
142
-
if (modelObj == null) {
143
-
return
144
-
}
145
-
try {
146
-
val objectRecognition = VNCoreMLRequest(modelObj) { request, error ->
147
-
val results = request?.results as? List<*> ?: return@VNCoreMLRequest
148
-
val recognized = results.filterIsInstance<VNRecognizedObjectObservation>()
149
-
processResults(recognized)
149
+
modelObj?.let {
150
+
try {
151
+
val objectRecognition = VNCoreMLRequest(it) { request, error ->
152
+
val results = request?.results as? List<*> ?: return@VNCoreMLRequest
153
+
val recognized = results.filterIsInstance<VNRecognizedObjectObservation>()
154
+
processResults(recognized)
155
+
}
156
+
requests = mutableListOf(objectRecognition)
157
+
} catch (e: Throwable) {
158
+
println("Vision setup error: ${e.message}")
150
159
}
151
-
requests = mutableListOf(objectRecognition)
152
-
} catch (e: Throwable) {
153
-
println("Vision setup error: ${e.message}")
154
160
}
155
161
}
156
162
···
346
352
fun analyseBufferForAll(
347
353
buffer: CVImageBufferRef?,
348
354
timestamp: Long,
349
-
mapPoint: (skeleton: Skeleton.SkeletonCoordinate) -> Skeleton.SkeletonCoordinate,
350
355
preview: AVCaptureVideoPreviewLayer?,
351
356
onObjectsProcessed: (List<AnalysisObject>) -> Unit,
352
357
onSkeletonProcessed: (Skeleton?) -> Unit
+4
-4
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.ios.kt
+4
-4
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.ios.kt
···
6
6
import com.performancecoachlab.posedetection.camera.drawAnalysisResults
7
7
import com.performancecoachlab.posedetection.camera.drawSkeleton
8
8
import com.performancecoachlab.posedetection.camera.toImageBitmap
9
-
import com.performancecoachlab.posedetection.custom.CustomObjectDetectorModels
9
+
import com.performancecoachlab.posedetection.custom.ObjectModel
10
10
import com.performancecoachlab.posedetection.skeleton.Skeleton
11
11
import kotlinx.cinterop.ExperimentalForeignApi
12
12
import kotlinx.coroutines.suspendCancellableCoroutine
···
35
35
}
36
36
}
37
37
38
-
actual class FrameAnalyser actual constructor() {
39
-
private val modelObj = CustomObjectDetectorModels.getInstance().model?.getModel()
38
+
actual class FrameAnalyser actual constructor(val model: ObjectModel?) {
39
+
private val modelObj = model?.getModel()
40
40
private val frameProcessor = FrameProcessor(modelObj)
41
41
42
42
@OptIn(ExperimentalForeignApi::class)
···
54
54
continuation.resume(AnalysisResult(poseResult, objectResults))
55
55
}
56
56
}
57
-
}
57
+
}
+21
-19
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
+21
-19
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
···
1
1
package com.nate.posedetection
2
2
3
3
import androidx.compose.animation.AnimatedContent
4
-
import androidx.compose.animation.AnimatedVisibility
5
4
import androidx.compose.foundation.Image
6
5
import androidx.compose.foundation.layout.Box
7
6
import androidx.compose.foundation.layout.Column
8
7
import androidx.compose.foundation.layout.fillMaxSize
9
8
import androidx.compose.foundation.layout.imePadding
10
9
import androidx.compose.foundation.layout.padding
11
-
import androidx.compose.foundation.layout.requiredHeight
12
-
import androidx.compose.foundation.layout.requiredWidth
13
-
import androidx.compose.foundation.layout.safeDrawing
14
10
import androidx.compose.material3.Button
15
11
import androidx.compose.material3.CircularProgressIndicator
16
12
import androidx.compose.material3.Tab
···
27
23
import androidx.compose.runtime.setValue
28
24
import androidx.compose.ui.Alignment
29
25
import androidx.compose.ui.Modifier
30
-
import androidx.compose.ui.geometry.Rect
31
26
import androidx.compose.ui.graphics.ImageBitmap
32
27
import androidx.compose.ui.layout.ContentScale
33
28
import androidx.compose.ui.unit.dp
34
29
import androidx.compose.ui.unit.sp
35
30
import chaintech.videoplayer.host.MediaPlayerHost
36
-
import chaintech.videoplayer.model.PlayerSpeed
37
31
import chaintech.videoplayer.model.ScreenResize
38
32
import chaintech.videoplayer.model.VideoPlayerConfig
39
33
import chaintech.videoplayer.ui.video.VideoPlayerComposable
40
34
import chaintech.videoplayer.util.RetrieveMediaDuration
41
35
import com.nate.posedetection.theme.AppTheme
42
36
import com.performancecoachlab.posedetection.camera.CameraView
43
-
import com.performancecoachlab.posedetection.camera.drawSkeleton
44
-
import com.performancecoachlab.posedetection.custom.CustomObjectDetectorModels
45
37
import com.performancecoachlab.posedetection.custom.CustomObjectRespository
46
38
import com.performancecoachlab.posedetection.custom.ModelPath
39
+
import com.performancecoachlab.posedetection.custom.initialiseObjectModel
47
40
import com.performancecoachlab.posedetection.encoding.VideoBuilder
48
41
import com.performancecoachlab.posedetection.encoding.createVideoBuilder
49
42
import com.performancecoachlab.posedetection.permissions.PermissionProvider
···
62
55
import io.github.vinceglb.filekit.filesDir
63
56
import io.github.vinceglb.filekit.path
64
57
import kotlinx.coroutines.Job
65
-
import kotlinx.coroutines.delay
66
58
import kotlinx.coroutines.launch
67
59
import kotlin.math.roundToLong
68
60
69
61
@Composable
70
62
internal fun App() = AppTheme {
71
-
var selectedTabIndex by remember { mutableStateOf(0) }
63
+
var selectedTabIndex by remember { mutableStateOf(1) }
72
64
val tabs = listOf("Camera Feed", "Recorded Video")
73
-
CustomObjectDetectorModels.init(
74
-
ModelPath("lite-model_efficientdet_lite2_detection_metadata_1.tflite", "YOLOv3FP16")
75
-
)
76
65
Column {
77
66
TabRow(selectedTabIndex = selectedTabIndex) {
78
67
tabs.forEachIndexed { index, title ->
···
140
129
var image by remember { mutableStateOf<InputFrame?>(null) }
141
130
val timeRange = Pair(0L, duration)
142
131
var frame by remember { mutableStateOf(timeRange.first) }
143
-
//val frameAnalyser = FrameAnalyser("4.tflite")
144
-
val frameAnalyser by remember { mutableStateOf(FrameAnalyser())}
132
+
val generalModel = initialiseObjectModel(
133
+
ModelPath(
134
+
"lite-model_efficientdet_lite2_detection_metadata_1.tflite",
135
+
"YOLOv3FP16"
136
+
)
137
+
)
138
+
val frameAnalyser by remember { mutableStateOf(FrameAnalyser(generalModel)) }
145
139
var bitmap by remember { mutableStateOf<ImageBitmap?>(null) }
146
140
val videoBuilder = remember { mutableStateOf<VideoBuilder?>(null) }
147
141
var isRecording by remember { mutableStateOf(false) }
···
204
198
startRecording()
205
199
}
206
200
if (firstFrameTimestamp == null) firstFrameTimestamp = frame.timestamp
207
-
val relativeTimestamp = frame.timestamp - (firstFrameTimestamp ?: frame.timestamp)
201
+
val relativeTimestamp =
202
+
frame.timestamp - (firstFrameTimestamp ?: frame.timestamp)
208
203
videoBuilder.value?.addFrame(it, relativeTimestamp)
209
204
bitmap = it
210
205
}
···
285
280
var permissionGranted by remember { mutableStateOf(false) }
286
281
var isRecording by remember { mutableStateOf(false) }
287
282
var path by remember { mutableStateOf("") }
283
+
val generalModel = initialiseObjectModel(
284
+
ModelPath(
285
+
"lite-model_efficientdet_lite2_detection_metadata_1.tflite",
286
+
"YOLOv3FP16"
287
+
)
288
+
)
288
289
PermissionProvider().apply {
289
290
if (!hasCameraPermission()) RequestCameraPermission(onGranted = {
290
291
permissionGranted = true
···
292
293
}
293
294
if (permissionGranted) {
294
295
Box(modifier = Modifier.fillMaxSize()) {
295
-
Column (modifier = Modifier.fillMaxSize()){
296
+
Column(modifier = Modifier.fillMaxSize()) {
296
297
androidx.compose.animation.AnimatedVisibility(path.isNotBlank()) {
297
298
val playerHost = remember(path) {
298
299
MediaPlayerHost(
···
323
324
skeletonRepository = skeletonRepository,
324
325
customObjectRepository = customObjectRespository,
325
326
drawSkeleton = true,
327
+
objectModel = generalModel,
326
328
drawObjects = true,
327
329
modifier = Modifier.weight(1f),
328
-
frontCamera = true,
330
+
frontCamera = false,
329
331
isRecording = isRecording,
330
332
onRecordToggled = { isRecording = it },
331
333
onVideoSaved = { path = it },
···
366
368
if (it.isNotEmpty()) {
367
369
it.forEach { obj ->
368
370
val l = "${obj.labels.maxByOrNull { it.confidence }?.text}"
369
-
println("Detected Objects: ${obj.labels}")
371
+
//println("Detected Objects: ${obj.labels}")
370
372
}
371
373
}
372
374
}