This repository has no description
refactor: ensure detection works correctly
author
nathan holland
date
(Jul 20, 2025, 10:03 PM +0100)
commit
44f42b75
44f42b754c44eecd70c3560cf9ed8866e2595612
parent
91a054a4
91a054a4d92827ba49fc9567ccb98544cb368508
+31
-45
+3
gradle/libs.versions.toml
+3
gradle/libs.versions.toml
···
11
11
media3CommonKtx = "1.7.1"
12
12
poseDetectionVersion = "18.0.0-beta5"
13
13
filekitCore = "0.10.0-beta04"
14
+
tensorflow = "0.4.4"
14
15
15
16
[libraries]
16
17
···
31
32
filekit-core = { module = "io.github.vinceglb:filekit-core", version.ref = "filekitCore" }
32
33
filekit-dialogs = { module = "io.github.vinceglb:filekit-dialogs", version.ref = "filekitCore" }
33
34
filekit-dialogs-compose = { module = "io.github.vinceglb:filekit-dialogs-compose", version.ref = "filekitCore" }
35
+
tensorflow-lite-support = { module = "org.tensorflow:tensorflow-lite-support", version.ref = "tensorflow" }
36
+
tensorflow-lite-task-vision = { module = "org.tensorflow:tensorflow-lite-task-vision", version.ref = "tensorflow" }
34
37
35
38
[plugins]
36
39
+1
-2
posedetection/build.gradle.kts
+1
-2
posedetection/build.gradle.kts
···
89
89
implementation(libs.pose.detection)
90
90
implementation(libs.pose.detection.common)
91
91
implementation(libs.androidx.media3.common.ktx)
92
-
implementation ("org.tensorflow:tensorflow-lite-support:0.4.4")
93
-
implementation("org.tensorflow:tensorflow-lite-task-vision:0.4.4")
92
+
implementation(libs.tensorflow.lite.task.vision)
94
93
}
95
94
96
95
}
+2
-2
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
+2
-2
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
···
75
75
val lifecycleOwner: LifecycleOwner = LocalLifecycleOwner.current
76
76
val previewView: PreviewView = remember { PreviewView(context) }
77
77
val executor = remember { Executors.newSingleThreadExecutor() }
78
-
val objDetectors = CustomObjectDetectorModels.getInstance().models
78
+
val objDetector = CustomObjectDetectorModels.getInstance().model
79
79
val scope = rememberCoroutineScope()
80
80
var firstFrameTimestamp: Long? = null
81
81
···
136
136
analysis.setAnalyzer(executor) { imageProxy ->
137
137
val timestamp = System.currentTimeMillis()
138
138
imageProxy.process(
139
-
objDetectors.firstOrNull()?.getDetector(), poseDetector, timestamp
139
+
objDetector?.getDetector(), poseDetector, timestamp
140
140
){
141
141
customObjectRepository.updateCustomObject(it.objects)
142
142
it.skeleton?.let { skel ->
+13
-19
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.android.kt
+13
-19
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.android.kt
···
4
4
import androidx.annotation.OptIn
5
5
import androidx.camera.core.ExperimentalGetImage
6
6
import androidx.camera.core.ImageProxy
7
-
import androidx.compose.runtime.getValue
8
-
import androidx.compose.runtime.mutableStateOf
9
-
import androidx.compose.runtime.remember
10
-
import androidx.compose.runtime.setValue
11
7
import androidx.compose.ui.geometry.Rect
12
-
import androidx.compose.ui.graphics.asImageBitmap
13
-
import androidx.core.graphics.toRect
14
8
import com.google.android.gms.tasks.Tasks
15
9
import com.google.mlkit.vision.common.InputImage
16
-
import com.google.mlkit.vision.objects.DetectedObject.Label
17
-
import com.google.mlkit.vision.objects.ObjectDetector
18
10
import com.google.mlkit.vision.pose.PoseDetector
19
11
import com.performancecoachlab.posedetection.recording.AnalysisObject
20
12
import com.performancecoachlab.posedetection.recording.AnalysisResult
···
39
31
onComplete: (AnalysisResult) -> Unit
40
32
) {
41
33
val tensorImage = TensorImage.fromBitmap(toBitmap())
42
-
val mlKitImage = image?.let{
34
+
val mlKitImage = image?.let {
43
35
InputImage.fromMediaImage(
44
36
it, imageInfo.rotationDegrees
45
37
)
···
76
68
)
77
69
}
78
70
79
-
private fun process (
71
+
private fun process(
80
72
tensorImage: TensorImage,
81
73
mlKitImage: InputImage?,
82
74
objectDetector: org.tensorflow.lite.task.vision.detector.ObjectDetector?,
···
86
78
height: Int,
87
79
onComplete: (AnalysisResult) -> Unit
88
80
) {
89
-
val objectsDetected = objectDetector?.detect(tensorImage)?.map { result->
81
+
val objectsDetected = objectDetector?.detect(tensorImage)?.map { result ->
90
82
AnalysisObject(
91
-
boundingBox = result.boundingBox.let { Rect(
92
-
left = it.left,
93
-
top = it.top,
94
-
right = it.right,
95
-
bottom = it.bottom
96
-
) },
83
+
boundingBox = result.boundingBox.let {
84
+
Rect(
85
+
left = it.left,
86
+
top = it.top,
87
+
right = it.right,
88
+
bottom = it.bottom
89
+
)
90
+
},
97
91
trackingId = 0,
98
92
labels = result.categories.map { category ->
99
93
com.performancecoachlab.posedetection.recording.Label(
···
102
96
)
103
97
}
104
98
)
105
-
}?: emptyList()
99
+
} ?: emptyList()
106
100
var skeleton: Skeleton? = null
107
101
val poseDetectionTask = mlKitImage?.let {
108
102
poseDetector.process(it).addOnSuccessListener { pose ->
···
111
105
println(e)
112
106
}
113
107
}
114
-
Tasks.whenAllComplete(listOfNotNull( poseDetectionTask)).addOnCompleteListener {
108
+
Tasks.whenAllComplete(listOfNotNull(poseDetectionTask)).addOnCompleteListener {
115
109
onComplete(
116
110
AnalysisResult(
117
111
skeleton = skeleton,
+1
-1
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.android.kt
+1
-1
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.android.kt
···
5
5
6
6
@Composable
7
7
actual fun initialiseObjectModel(modelPath: ModelPath): ObjectModel {
8
-
val options = org.tensorflow.lite.task.vision.detector.ObjectDetector.ObjectDetectorOptions.builder().setMaxResults(10).setScoreThreshold(0.1f).build()
8
+
val options = org.tensorflow.lite.task.vision.detector.ObjectDetector.ObjectDetectorOptions.builder().setMaxResults(5).setScoreThreshold(0f).build()
9
9
val detector = org.tensorflow.lite.task.vision.detector.ObjectDetector.createFromFileAndOptions(
10
10
LocalContext.current,
11
11
modelPath.androidModelPath,
+2
-2
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.android.kt
+2
-2
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.android.kt
···
31
31
private val options =
32
32
PoseDetectorOptions.Builder().setDetectorMode(PoseDetectorOptions.STREAM_MODE).build()
33
33
private val poseDetector = PoseDetection.getClient(options)
34
-
private val objDetectors = CustomObjectDetectorModels.getInstance().models
34
+
private val objDetector = CustomObjectDetectorModels.getInstance().model?.getDetector()
35
35
actual suspend fun analyseFrame(inputFrame: InputFrame): AnalysisResult =
36
36
suspendCancellableCoroutine { continuation ->
37
37
inputFrame.bitmap.process(
38
-
objDetectors.firstOrNull()?.getDetector(), poseDetector, inputFrame.timestamp
38
+
objDetector, poseDetector, inputFrame.timestamp
39
39
) { result ->
40
40
continuation.resume(result)
41
41
}
+5
-8
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.kt
+5
-8
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.kt
···
4
4
import kotlin.concurrent.Volatile
5
5
6
6
data class ModelPath(
7
-
val id: Enum<*>,
8
7
val androidModelPath: String? = null,
9
8
val iosModelPath: String? = null,
10
9
)
11
10
12
11
class CustomObjectDetectorModels private constructor(
13
-
val models: List<ObjectModel>,
12
+
val model: ObjectModel?
14
13
) {
15
14
companion object {
16
15
@Volatile
17
-
private var instance = CustomObjectDetectorModels(emptyList())
16
+
private var instance = CustomObjectDetectorModels(null)
18
17
19
18
@Composable
20
-
fun init(paths: List<ModelPath>) {
21
-
val models = paths.map { modelPaths ->
22
-
initialiseObjectModel(modelPaths)
23
-
}
24
-
instance = CustomObjectDetectorModels(models = models)
19
+
fun init(path: ModelPath) {
20
+
val model = initialiseObjectModel(path)
21
+
instance = CustomObjectDetectorModels(model = model)
25
22
}
26
23
27
24
fun getInstance(): CustomObjectDetectorModels = instance
+1
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
+1
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
···
439
439
}
440
440
}
441
441
442
-
private val objectDetector = CustomObjectDetectorModels.getInstance().models.firstOrNull()?.getModel()
442
+
private val objectDetector = CustomObjectDetectorModels.getInstance().model?.getModel()
443
443
private val frameProcessor = FrameProcessor(objectDetector)
444
444
445
445
@OptIn(ExperimentalForeignApi::class, NativeRuntimeApi::class)
+1
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.ios.kt
+1
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.ios.kt
···
35
35
}
36
36
37
37
actual class FrameAnalyser actual constructor() {
38
-
private val modelObj = CustomObjectDetectorModels.getInstance().models.firstOrNull()?.getModel()
38
+
private val modelObj = CustomObjectDetectorModels.getInstance().model?.getModel()
39
39
private val frameProcessor = FrameProcessor(modelObj)
40
40
41
41
@OptIn(ExperimentalForeignApi::class)
sample/composeApp/src/androidMain/assets/latest.tflite
sample/composeApp/src/androidMain/assets/latest.tflite
This is a binary file and will not be displayed.
sample/composeApp/src/androidMain/assets/lite-model_efficientdet_lite2_detection_metadata_1.tflite
sample/composeApp/src/androidMain/assets/lite-model_efficientdet_lite2_detection_metadata_1.tflite
This is a binary file and will not be displayed.
+2
-9
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
+2
-9
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
···
62
62
import kotlinx.coroutines.launch
63
63
import kotlin.math.roundToLong
64
64
65
-
66
-
enum class CustomModels{
67
-
DEFAULT
68
-
}
69
-
70
65
@Composable
71
66
internal fun App() = AppTheme {
72
-
var selectedTabIndex by remember { mutableStateOf(0) }
67
+
var selectedTabIndex by remember { mutableStateOf(1) }
73
68
val tabs = listOf("Camera Feed", "Recorded Video")
74
69
75
70
CustomObjectDetectorModels.init(
76
-
listOf(ModelPath(
77
-
CustomModels.DEFAULT,"latest.tflite", "YOLOv3FP16"
78
-
))
71
+
ModelPath("lite-model_efficientdet_lite2_detection_metadata_1.tflite", "YOLOv3FP16")
79
72
)
80
73
81
74