This repository has no description
0

Configure Feed

Select the types of activity you want to include in your feed.

refactor: ensure detection works correctly

+31 -45
+3
gradle/libs.versions.toml
··· 11 11 media3CommonKtx = "1.7.1" 12 12 poseDetectionVersion = "18.0.0-beta5" 13 13 filekitCore = "0.10.0-beta04" 14 + tensorflow = "0.4.4" 14 15 15 16 [libraries] 16 17 ··· 31 32 filekit-core = { module = "io.github.vinceglb:filekit-core", version.ref = "filekitCore" } 32 33 filekit-dialogs = { module = "io.github.vinceglb:filekit-dialogs", version.ref = "filekitCore" } 33 34 filekit-dialogs-compose = { module = "io.github.vinceglb:filekit-dialogs-compose", version.ref = "filekitCore" } 35 + tensorflow-lite-support = { module = "org.tensorflow:tensorflow-lite-support", version.ref = "tensorflow" } 36 + tensorflow-lite-task-vision = { module = "org.tensorflow:tensorflow-lite-task-vision", version.ref = "tensorflow" } 34 37 35 38 [plugins] 36 39
+1 -2
posedetection/build.gradle.kts
··· 89 89 implementation(libs.pose.detection) 90 90 implementation(libs.pose.detection.common) 91 91 implementation(libs.androidx.media3.common.ktx) 92 - implementation ("org.tensorflow:tensorflow-lite-support:0.4.4") 93 - implementation("org.tensorflow:tensorflow-lite-task-vision:0.4.4") 92 + implementation(libs.tensorflow.lite.task.vision) 94 93 } 95 94 96 95 }
+2 -2
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
··· 75 75 val lifecycleOwner: LifecycleOwner = LocalLifecycleOwner.current 76 76 val previewView: PreviewView = remember { PreviewView(context) } 77 77 val executor = remember { Executors.newSingleThreadExecutor() } 78 - val objDetectors = CustomObjectDetectorModels.getInstance().models 78 + val objDetector = CustomObjectDetectorModels.getInstance().model 79 79 val scope = rememberCoroutineScope() 80 80 var firstFrameTimestamp: Long? = null 81 81 ··· 136 136 analysis.setAnalyzer(executor) { imageProxy -> 137 137 val timestamp = System.currentTimeMillis() 138 138 imageProxy.process( 139 - objDetectors.firstOrNull()?.getDetector(), poseDetector, timestamp 139 + objDetector?.getDetector(), poseDetector, timestamp 140 140 ){ 141 141 customObjectRepository.updateCustomObject(it.objects) 142 142 it.skeleton?.let { skel ->
+13 -19
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.android.kt
··· 4 4 import androidx.annotation.OptIn 5 5 import androidx.camera.core.ExperimentalGetImage 6 6 import androidx.camera.core.ImageProxy 7 - import androidx.compose.runtime.getValue 8 - import androidx.compose.runtime.mutableStateOf 9 - import androidx.compose.runtime.remember 10 - import androidx.compose.runtime.setValue 11 7 import androidx.compose.ui.geometry.Rect 12 - import androidx.compose.ui.graphics.asImageBitmap 13 - import androidx.core.graphics.toRect 14 8 import com.google.android.gms.tasks.Tasks 15 9 import com.google.mlkit.vision.common.InputImage 16 - import com.google.mlkit.vision.objects.DetectedObject.Label 17 - import com.google.mlkit.vision.objects.ObjectDetector 18 10 import com.google.mlkit.vision.pose.PoseDetector 19 11 import com.performancecoachlab.posedetection.recording.AnalysisObject 20 12 import com.performancecoachlab.posedetection.recording.AnalysisResult ··· 39 31 onComplete: (AnalysisResult) -> Unit 40 32 ) { 41 33 val tensorImage = TensorImage.fromBitmap(toBitmap()) 42 - val mlKitImage = image?.let{ 34 + val mlKitImage = image?.let { 43 35 InputImage.fromMediaImage( 44 36 it, imageInfo.rotationDegrees 45 37 ) ··· 76 68 ) 77 69 } 78 70 79 - private fun process ( 71 + private fun process( 80 72 tensorImage: TensorImage, 81 73 mlKitImage: InputImage?, 82 74 objectDetector: org.tensorflow.lite.task.vision.detector.ObjectDetector?, ··· 86 78 height: Int, 87 79 onComplete: (AnalysisResult) -> Unit 88 80 ) { 89 - val objectsDetected = objectDetector?.detect(tensorImage)?.map { result-> 81 + val objectsDetected = objectDetector?.detect(tensorImage)?.map { result -> 90 82 AnalysisObject( 91 - boundingBox = result.boundingBox.let { Rect( 92 - left = it.left, 93 - top = it.top, 94 - right = it.right, 95 - bottom = it.bottom 96 - ) }, 83 + boundingBox = result.boundingBox.let { 84 + Rect( 85 + left = it.left, 86 + top = it.top, 87 + right = it.right, 88 + bottom = it.bottom 89 + ) 90 + }, 97 91 trackingId = 0, 98 92 labels = result.categories.map { category -> 99 93 com.performancecoachlab.posedetection.recording.Label( ··· 102 96 ) 103 97 } 104 98 ) 105 - }?: emptyList() 99 + } ?: emptyList() 106 100 var skeleton: Skeleton? = null 107 101 val poseDetectionTask = mlKitImage?.let { 108 102 poseDetector.process(it).addOnSuccessListener { pose -> ··· 111 105 println(e) 112 106 } 113 107 } 114 - Tasks.whenAllComplete(listOfNotNull( poseDetectionTask)).addOnCompleteListener { 108 + Tasks.whenAllComplete(listOfNotNull(poseDetectionTask)).addOnCompleteListener { 115 109 onComplete( 116 110 AnalysisResult( 117 111 skeleton = skeleton,
+1 -1
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.android.kt
··· 5 5 6 6 @Composable 7 7 actual fun initialiseObjectModel(modelPath: ModelPath): ObjectModel { 8 - val options = org.tensorflow.lite.task.vision.detector.ObjectDetector.ObjectDetectorOptions.builder().setMaxResults(10).setScoreThreshold(0.1f).build() 8 + val options = org.tensorflow.lite.task.vision.detector.ObjectDetector.ObjectDetectorOptions.builder().setMaxResults(5).setScoreThreshold(0f).build() 9 9 val detector = org.tensorflow.lite.task.vision.detector.ObjectDetector.createFromFileAndOptions( 10 10 LocalContext.current, 11 11 modelPath.androidModelPath,
+2 -2
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.android.kt
··· 31 31 private val options = 32 32 PoseDetectorOptions.Builder().setDetectorMode(PoseDetectorOptions.STREAM_MODE).build() 33 33 private val poseDetector = PoseDetection.getClient(options) 34 - private val objDetectors = CustomObjectDetectorModels.getInstance().models 34 + private val objDetector = CustomObjectDetectorModels.getInstance().model?.getDetector() 35 35 actual suspend fun analyseFrame(inputFrame: InputFrame): AnalysisResult = 36 36 suspendCancellableCoroutine { continuation -> 37 37 inputFrame.bitmap.process( 38 - objDetectors.firstOrNull()?.getDetector(), poseDetector, inputFrame.timestamp 38 + objDetector, poseDetector, inputFrame.timestamp 39 39 ) { result -> 40 40 continuation.resume(result) 41 41 }
+5 -8
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.kt
··· 4 4 import kotlin.concurrent.Volatile 5 5 6 6 data class ModelPath( 7 - val id: Enum<*>, 8 7 val androidModelPath: String? = null, 9 8 val iosModelPath: String? = null, 10 9 ) 11 10 12 11 class CustomObjectDetectorModels private constructor( 13 - val models: List<ObjectModel>, 12 + val model: ObjectModel? 14 13 ) { 15 14 companion object { 16 15 @Volatile 17 - private var instance = CustomObjectDetectorModels(emptyList()) 16 + private var instance = CustomObjectDetectorModels(null) 18 17 19 18 @Composable 20 - fun init(paths: List<ModelPath>) { 21 - val models = paths.map { modelPaths -> 22 - initialiseObjectModel(modelPaths) 23 - } 24 - instance = CustomObjectDetectorModels(models = models) 19 + fun init(path: ModelPath) { 20 + val model = initialiseObjectModel(path) 21 + instance = CustomObjectDetectorModels(model = model) 25 22 } 26 23 27 24 fun getInstance(): CustomObjectDetectorModels = instance
+1 -1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraEngine.kt
··· 439 439 } 440 440 } 441 441 442 - private val objectDetector = CustomObjectDetectorModels.getInstance().models.firstOrNull()?.getModel() 442 + private val objectDetector = CustomObjectDetectorModels.getInstance().model?.getModel() 443 443 private val frameProcessor = FrameProcessor(objectDetector) 444 444 445 445 @OptIn(ExperimentalForeignApi::class, NativeRuntimeApi::class)
+1 -1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.ios.kt
··· 35 35 } 36 36 37 37 actual class FrameAnalyser actual constructor() { 38 - private val modelObj = CustomObjectDetectorModels.getInstance().models.firstOrNull()?.getModel() 38 + private val modelObj = CustomObjectDetectorModels.getInstance().model?.getModel() 39 39 private val frameProcessor = FrameProcessor(modelObj) 40 40 41 41 @OptIn(ExperimentalForeignApi::class)
sample/composeApp/src/androidMain/assets/latest.tflite

This is a binary file and will not be displayed.

sample/composeApp/src/androidMain/assets/lite-model_efficientdet_lite2_detection_metadata_1.tflite

This is a binary file and will not be displayed.

+2 -9
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
··· 62 62 import kotlinx.coroutines.launch 63 63 import kotlin.math.roundToLong 64 64 65 - 66 - enum class CustomModels{ 67 - DEFAULT 68 - } 69 - 70 65 @Composable 71 66 internal fun App() = AppTheme { 72 - var selectedTabIndex by remember { mutableStateOf(0) } 67 + var selectedTabIndex by remember { mutableStateOf(1) } 73 68 val tabs = listOf("Camera Feed", "Recorded Video") 74 69 75 70 CustomObjectDetectorModels.init( 76 - listOf(ModelPath( 77 - CustomModels.DEFAULT,"latest.tflite", "YOLOv3FP16" 78 - )) 71 + ModelPath("lite-model_efficientdet_lite2_detection_metadata_1.tflite", "YOLOv3FP16") 79 72 ) 80 73 81 74