This repository has no description
refactor: object detection experiments
author
nathan holland
date
(Jun 7, 2025, 12:48 PM +0300)
commit
7ce85ada
7ce85adaa8fd6288896e747cf5624216a4f2cb1d
parent
996564d2
996564d262bf39080d8e5908f0ad2ebe6b5cbb91
+81
-12
+4
-1
posedetection/build.gradle.kts
+4
-1
posedetection/build.gradle.kts
···
1
+
import com.android.build.api.dsl.AaptOptions
2
+
import com.android.build.api.dsl.AndroidResources
1
3
import com.vanniktech.maven.publish.SonatypeHost
2
4
import org.jetbrains.compose.ExperimentalComposeLibrary
3
5
import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSetTree
···
87
89
implementation(libs.pose.detection)
88
90
implementation(libs.pose.detection.common)
89
91
implementation(libs.androidx.media3.common.ktx)
92
+
implementation("com.google.mlkit:object-detection-custom:17.0.2")
93
+
implementation ("org.tensorflow:tensorflow-lite-support:0.4.4")
90
94
}
91
95
92
96
}
···
95
99
android {
96
100
namespace = "com.performancecoachlab.posedetection"
97
101
compileSdk = 35
98
-
99
102
defaultConfig {
100
103
minSdk = 21
101
104
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
posedetection/src/androidMain/assets/2.tflite
posedetection/src/androidMain/assets/2.tflite
This is a binary file and will not be displayed.
+17
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
+17
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
···
36
36
import com.google.mlkit.vision.pose.PoseDetection
37
37
import com.google.mlkit.vision.pose.PoseLandmark
38
38
import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions
39
+
import com.performancecoachlab.posedetection.objects.createObjectDetector
39
40
import com.performancecoachlab.posedetection.skeleton.Skeleton
40
41
import com.performancecoachlab.posedetection.skeleton.SkeletonRepository
41
42
import java.util.concurrent.Executors
···
58
59
val lifecycleOwner: LifecycleOwner = LocalLifecycleOwner.current
59
60
val previewView: PreviewView = remember { PreviewView(context) }
60
61
val executor = remember { Executors.newSingleThreadExecutor() }
62
+
val objectDetector = createObjectDetector()
61
63
LaunchedEffect(lifecycleOwner) {
62
64
cameraController.bindToLifecycle(lifecycleOwner)
63
65
cameraController.cameraSelector =
···
70
72
val img = InputImage.fromMediaImage(
71
73
image, imageProxy.imageInfo.rotationDegrees
72
74
)
75
+
objectDetector.process(img).addOnSuccessListener {
76
+
for (detectedObject in it) {
77
+
val boundingBox = detectedObject.boundingBox
78
+
val trackingId = detectedObject.trackingId
79
+
for (label in detectedObject.labels) {
80
+
val text = label.text
81
+
val index = label.index
82
+
val confidence = label.confidence
83
+
println("Detected object: $text, Index: $index, Confidence: $confidence, BoundingBox: $boundingBox, TrackingId: $trackingId")
84
+
}
85
+
}
86
+
}.addOnFailureListener { e ->
87
+
// Handle object detection failure
88
+
println(e)
89
+
}
73
90
poseDetector.process(img).addOnSuccessListener { pose ->
74
91
skeleton = skeleton(pose, timestamp, img.width, img.height)
75
92
skeletonRepository.updateSkeleton(skeleton.mirror())
+20
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/objects/createObjectDetector.kt
+20
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/objects/createObjectDetector.kt
···
1
+
package com.performancecoachlab.posedetection.objects
2
+
3
+
import com.google.mlkit.common.model.LocalModel
4
+
import com.google.mlkit.vision.objects.ObjectDetection
5
+
import com.google.mlkit.vision.objects.ObjectDetector
6
+
import com.google.mlkit.vision.objects.custom.CustomObjectDetectorOptions
7
+
8
+
fun createObjectDetector(): ObjectDetector {
9
+
val localModel = LocalModel.Builder()
10
+
.setAssetFilePath("2.tflite")
11
+
.build()
12
+
val customObjectDetectorOptions =
13
+
CustomObjectDetectorOptions.Builder(localModel)
14
+
.setDetectorMode(CustomObjectDetectorOptions.STREAM_MODE)
15
+
.enableClassification()
16
+
.setClassificationConfidenceThreshold(0.5f)
17
+
.setMaxPerObjectLabelCount(3)
18
+
.build()
19
+
return ObjectDetection.getClient(customObjectDetectorOptions)
20
+
}
+40
-11
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.android.kt
+40
-11
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.android.kt
···
7
7
import com.google.mlkit.vision.pose.PoseDetection
8
8
import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions
9
9
import com.performancecoachlab.posedetection.camera.drawSkeleton
10
-
import com.performancecoachlab.posedetection.camera.rotate
11
10
import com.performancecoachlab.posedetection.camera.skeleton
11
+
import com.performancecoachlab.posedetection.objects.createObjectDetector
12
12
import com.performancecoachlab.posedetection.skeleton.Skeleton
13
13
import kotlinx.coroutines.suspendCancellableCoroutine
14
14
import kotlin.coroutines.resume
15
+
import org.tensorflow.lite.support.tensorbuffer.TensorBuffer
16
+
import org.tensorflow.lite.DataType
17
+
import java.nio.ByteBuffer
18
+
import java.nio.ByteOrder
15
19
16
20
actual class InputFrame(val bitmap: Bitmap, actual val timestamp: Long) {
17
21
actual fun toImageBitmap(): ImageBitmap {
···
27
31
private val options =
28
32
PoseDetectorOptions.Builder().setDetectorMode(PoseDetectorOptions.STREAM_MODE).build()
29
33
private val poseDetector = PoseDetection.getClient(options)
34
+
private val objectDetector = createObjectDetector()
35
+
actual suspend fun analyseFrame(inputFrame: InputFrame): Skeleton? =
36
+
suspendCancellableCoroutine { continuation ->
37
+
val img = InputImage.fromBitmap(inputFrame.bitmap, 0)
38
+
var poseResult: Skeleton? = null
39
+
var objectResult: Any? = null
40
+
var completed = 0
30
41
31
-
actual suspend fun analyseFrame(inputFrame: InputFrame): Skeleton? {
32
-
val img = InputImage.fromBitmap(inputFrame.bitmap, 0)
33
-
return suspendCancellableCoroutine { continuation ->
34
-
poseDetector.process(img)
35
-
.addOnSuccessListener { pose ->
36
-
val skeleton = skeleton(pose, inputFrame.timestamp, img.width, img.height)
37
-
continuation.resume(skeleton)
42
+
fun tryResume() {
43
+
completed++
44
+
if (completed == 2) {
45
+
continuation.resume(poseResult)
38
46
}
39
-
.addOnFailureListener {
40
-
continuation.resume(null)
47
+
}
48
+
49
+
objectDetector.process(img).addOnSuccessListener {
50
+
for (detectedObject in it) {
51
+
val boundingBox = detectedObject.boundingBox
52
+
val trackingId = detectedObject.trackingId
53
+
for (label in detectedObject.labels) {
54
+
val text = label.text
55
+
val index = label.index
56
+
val confidence = label.confidence
57
+
println("Detected object: $text, Index: $index, Confidence: $confidence, BoundingBox: $boundingBox, TrackingId: $trackingId")
58
+
}
41
59
}
60
+
tryResume()
61
+
}.addOnFailureListener {
62
+
val e = it
63
+
tryResume()
64
+
}
65
+
66
+
poseDetector.process(img).addOnSuccessListener { pose ->
67
+
poseResult = skeleton(pose, inputFrame.timestamp, img.width, img.height)
68
+
tryResume()
69
+
}.addOnFailureListener {
70
+
tryResume()
71
+
}
42
72
}
43
-
}
44
73
}