This repository has no description
fix: pose detection in frame analysis
author
nate
date
(Feb 5, 2026, 9:57 AM +0200)
commit
9ff107e5
9ff107e5cd8362d0fc475d622ca8b44b6c271115
parent
beb1e6fc
beb1e6fc56eec4c21a533288f9dc21a9058597d7
+167
-19
+40
-11
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
+40
-11
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
···
68
68
import androidx.camera.core.CameraSelector
69
69
import co.touchlab.kermit.Logger
70
70
import java.util.concurrent.atomic.AtomicBoolean
71
+
import java.util.concurrent.atomic.AtomicLong
71
72
72
73
// Data class to hold recording state for each recording ID
73
74
data class RecordingSlot(
···
134
135
var analysisSize by remember { mutableStateOf<Size?>(null) }
135
136
var latestSkeleton by remember { mutableStateOf<Skeleton?>(null) }
136
137
var latestObjects by remember { mutableStateOf<List<AnalysisObject>>(emptyList()) }
138
+
139
+
// Throttle detectors to avoid doing heavy work every frame.
140
+
// Tune these to balance smoothness vs CPU usage.
141
+
val objectIntervalMs = 66L // ~10 FPS
142
+
val poseIntervalMs = 66L // ~10 FPS
143
+
val lastObjectRunAtMs = remember { AtomicLong(0L) }
144
+
val lastPoseRunAtMs = remember { AtomicLong(0L) }
137
145
138
146
val options = PoseDetectorOptions.Builder()
139
147
.setDetectorMode(PoseDetectorOptions.STREAM_MODE)
···
275
283
return@setAnalyzer
276
284
}
277
285
278
-
val timestamp = System.currentTimeMillis()
286
+
val now = System.currentTimeMillis()
279
287
val area = focus
280
-
val poseClient = if (currentDetectMode.doPose()) poseDetector else null
281
-
val objectClient = if (currentDetectMode.doObject()) objectDetector?.getDetector() else null
288
+
289
+
val shouldRunObject = currentDetectMode.doObject() &&
290
+
(now - lastObjectRunAtMs.get() >= objectIntervalMs)
291
+
val shouldRunPose = currentDetectMode.doPose() &&
292
+
(now - lastPoseRunAtMs.get() >= poseIntervalMs)
293
+
294
+
// If neither detector is scheduled to run, just close quickly and reuse last results.
295
+
if (!shouldRunObject && !shouldRunPose) {
296
+
// still release the gate; no repositories updated
297
+
imageProxy.close()
298
+
gate.exit()
299
+
return@setAnalyzer
300
+
}
301
+
302
+
if (shouldRunObject) lastObjectRunAtMs.set(now)
303
+
if (shouldRunPose) lastPoseRunAtMs.set(now)
304
+
305
+
val poseClient = if (shouldRunPose) poseDetector else null
306
+
val objectClient = if (shouldRunObject) objectDetector?.getDetector() else null
282
307
val rotationDegrees = imageProxy.imageInfo.rotationDegrees
283
308
284
-
imageProxy.process(objectClient, poseClient, timestamp, area) { analysisResult, frameBitmap ->
309
+
imageProxy.process(objectClient, poseClient, now, area) { analysisResult, frameBitmap ->
285
310
try {
286
-
// Keep repositories as before.
287
-
customObjectRepository.updateCustomObject(analysisResult.objects)
288
-
analysisResult.skeleton?.let { skeletonRepository.updateSkeleton(it) }
311
+
// Only update repositories/results for detectors that actually ran.
312
+
if (shouldRunObject) {
313
+
customObjectRepository.updateCustomObject(analysisResult.objects)
314
+
latestObjects = analysisResult.objects
315
+
}
316
+
if (shouldRunPose) {
317
+
analysisResult.skeleton?.let { skeletonRepository.updateSkeleton(it) }
318
+
latestSkeleton = if (drawSkeleton) analysisResult.skeleton else null
319
+
}
289
320
290
-
// Update UI overlay state.
291
-
latestSkeleton = if (drawSkeleton) analysisResult.skeleton else null
292
-
latestObjects = analysisResult.objects
321
+
// Size is based on the frame bitmap we got back.
293
322
analysisSize = androidx.compose.ui.geometry.Size(
294
323
width = frameBitmap.width.toFloat(),
295
324
height = frameBitmap.height.toFloat()
···
312
341
// Only convert to ImageBitmap when we're actually recording.
313
342
if (activeRecordings.isNotEmpty()) {
314
343
val inbmp = frameBitmap.asImageBitmap()
315
-
addFrameToActiveRecordings(inbmp, timestamp)
344
+
addFrameToActiveRecordings(inbmp, now)
316
345
}
317
346
} finally {
318
347
gate.exit()
+7
-1
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.android.kt
+7
-1
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.android.kt
···
258
258
onComplete = onComplete
259
259
)
260
260
}
261
-
private fun process(
261
+
262
+
// Core processing used by both live camera (ImageProxy) and offline frames (FrameAnalyser).
263
+
fun process(
262
264
tensorImage: TensorImage?,
263
265
mlKitImage: InputImage?,
264
266
objectDetector: Interpreter?,
···
343
345
val landmarks = pose.allPoseLandmarks.size
344
346
Logger.d { "MLKit pose landmarks=$landmarks" }
345
347
skeleton(pose, timestamp, width, height)
348
+
}.onFailure { t ->
349
+
Logger.e(t) { "MLKit poseDetector.process failed" }
346
350
}.getOrNull()
347
351
} else null
348
352
···
531
535
val cls: Int,
532
536
val clsName: String
533
537
)
538
+
539
+
534
540
535
541
536
542
+118
-5
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.android.kt
+118
-5
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.android.kt
···
1
1
package com.performancecoachlab.posedetection.recording
2
2
3
3
import android.graphics.Bitmap
4
+
import android.graphics.Canvas
5
+
import android.graphics.Paint
4
6
import androidx.compose.ui.geometry.Rect
5
7
import androidx.compose.ui.graphics.ImageBitmap
6
8
import androidx.compose.ui.graphics.asImageBitmap
9
+
import com.google.mlkit.vision.common.InputImage
7
10
import com.google.mlkit.vision.pose.PoseDetection
8
11
import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions
12
+
import com.performancecoachlab.posedetection.camera.applyFocusAreaMaskPooled
9
13
import com.performancecoachlab.posedetection.camera.drawAnalysisResults
10
14
import com.performancecoachlab.posedetection.camera.drawSkeleton
11
-
import com.performancecoachlab.posedetection.camera.process
12
15
import com.performancecoachlab.posedetection.custom.ObjectModel
13
16
import com.performancecoachlab.posedetection.skeleton.Skeleton
14
17
import kotlinx.coroutines.suspendCancellableCoroutine
18
+
import org.tensorflow.lite.DataType
19
+
import org.tensorflow.lite.support.common.ops.CastOp
20
+
import org.tensorflow.lite.support.common.ops.NormalizeOp
21
+
import org.tensorflow.lite.support.image.ImageProcessor
22
+
import org.tensorflow.lite.support.image.TensorImage
15
23
import kotlin.coroutines.resume
24
+
import kotlin.math.max
25
+
import androidx.core.graphics.createBitmap
26
+
import co.touchlab.kermit.Logger
27
+
import com.google.android.gms.tasks.Tasks
28
+
import com.performancecoachlab.posedetection.camera.process
29
+
import kotlinx.coroutines.Dispatchers
30
+
import kotlinx.coroutines.launch
16
31
17
32
actual class InputFrame(val bitmap: Bitmap, actual val timestamp: Long) {
18
33
actual fun toImageBitmap(): ImageBitmap {
···
52
67
PoseDetectorOptions.Builder().setDetectorMode(PoseDetectorOptions.STREAM_MODE).build()
53
68
private val poseDetector = PoseDetection.getClient(options)
54
69
private val objDetector = model?.getDetector()
70
+
71
+
private val imageProcessor = ImageProcessor.Builder()
72
+
.add(NormalizeOp(0f, 255f))
73
+
.add(CastOp(DataType.FLOAT32))
74
+
.build()
75
+
76
+
private var tensorBitmap: Bitmap? = null
77
+
private var tensorW: Int = 0
78
+
private var tensorH: Int = 0
79
+
80
+
private fun ensureTensorBitmap(width: Int, height: Int): Bitmap {
81
+
val existing = tensorBitmap
82
+
return if (existing != null && !existing.isRecycled && tensorW == width && tensorH == height) {
83
+
existing
84
+
} else {
85
+
androidx.core.graphics.createBitmap(width, height, Bitmap.Config.ARGB_8888).also {
86
+
tensorBitmap = it
87
+
tensorW = width
88
+
tensorH = height
89
+
}
90
+
}
91
+
}
92
+
private fun toPoseBitmap(src: Bitmap): Bitmap {
93
+
val argb8888 = if (src.config == Bitmap.Config.ARGB_8888) src else src.copy(Bitmap.Config.ARGB_8888, false)
94
+
95
+
val minSide = 480
96
+
val w = argb8888.width
97
+
val h = argb8888.height
98
+
val scale = max(minSide.toFloat() / w.toFloat(), minSide.toFloat() / h.toFloat())
99
+
if (scale <= 1f) return argb8888
100
+
101
+
val outW = (w * scale).toInt().coerceAtLeast(1)
102
+
val outH = (h * scale).toInt().coerceAtLeast(1)
103
+
val resized = createBitmap(outW, outH)
104
+
Canvas(resized).drawBitmap(
105
+
argb8888,
106
+
null,
107
+
android.graphics.Rect(0, 0, outW, outH),
108
+
Paint(Paint.FILTER_BITMAP_FLAG)
109
+
)
110
+
return resized
111
+
}
112
+
55
113
actual suspend fun analyseFrame(inputFrame: InputFrame, focusArea: Rect?): AnalysisResult =
56
114
suspendCancellableCoroutine { continuation ->
57
-
inputFrame.bitmap.process(
58
-
objDetector, poseDetector, inputFrame.timestamp,focusArea
59
-
) { result, bitmap ->
60
-
continuation.resume(result)
115
+
val interpreter = objDetector
116
+
117
+
// Pose: keep original resolution (optionally masked) for ML Kit.
118
+
val masked = inputFrame.bitmap.applyFocusAreaMaskPooled(focusArea, 0)
119
+
val mlKitImage = InputImage.fromBitmap(masked, 0)
120
+
121
+
// Object: resize to model input size, then normalize.
122
+
val tensorImage: TensorImage? = interpreter?.let { tfl ->
123
+
val shape = tfl.getInputTensor(0)?.shape()
124
+
var w = 0
125
+
var h = 0
126
+
if (shape != null) {
127
+
w = shape[1]
128
+
h = shape[2]
129
+
if (shape[1] == 3) {
130
+
w = shape[2]
131
+
h = shape[3]
132
+
}
133
+
}
134
+
if (w <= 0 || h <= 0) null else {
135
+
val dst = ensureTensorBitmap(w, h)
136
+
// Draw scaled into dst (no new allocations).
137
+
android.graphics.Canvas(dst).drawBitmap(
138
+
inputFrame.bitmap,
139
+
null,
140
+
android.graphics.Rect(0, 0, w, h),
141
+
android.graphics.Paint(android.graphics.Paint.FILTER_BITMAP_FLAG)
142
+
)
143
+
TensorImage(DataType.FLOAT32).also { ti -> ti.load(dst) }.let(imageProcessor::process)
144
+
}
145
+
}
146
+
continuation.context.let { ctx ->
147
+
kotlinx.coroutines.CoroutineScope(ctx).launch(Dispatchers.Default) {
148
+
try {
149
+
process(
150
+
tensorImage = tensorImage,
151
+
mlKitImage = mlKitImage,
152
+
objectDetector = interpreter,
153
+
poseDetector = poseDetector,
154
+
timestamp = inputFrame.timestamp,
155
+
width = inputFrame.bitmap.width,
156
+
height = inputFrame.bitmap.height,
157
+
bitmap = inputFrame.bitmap,
158
+
onComplete = { result, _ ->
159
+
if (continuation.isActive) continuation.resume(result)
160
+
}
161
+
)
162
+
} catch (t: Throwable) {
163
+
Logger.e(t) { "Offline analyseFrame: process() crashed" }
164
+
if (continuation.isActive) {
165
+
continuation.resume(
166
+
AnalysisResult(
167
+
skeleton = null,
168
+
objects = emptyList()
169
+
)
170
+
)
171
+
}
172
+
}
173
+
}
61
174
}
62
175
}
63
176
}
+2
-2
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
+2
-2
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
···
80
80
import kotlin.time.Clock
81
81
import kotlin.time.ExperimentalTime
82
82
83
-
val androidPath = "hoops.tflite"
84
-
val iosPath = "YOLOv3FP16"
83
+
val androidPath = "basketballs_n1.tflite"
84
+
val iosPath = "basketballs_n1"
85
85
@Composable
86
86
internal fun App() = AppTheme {
87
87
var selectedTabIndex by remember { mutableStateOf(0) }