This repository has no description
0

Configure Feed

Select the types of activity you want to include in your feed.

fix: pose detection broken

+286 -90
+156 -60
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
··· 1 1 package com.performancecoachlab.posedetection.camera 2 2 3 3 import android.graphics.Bitmap 4 + import android.graphics.Matrix 4 5 import androidx.annotation.OptIn 5 6 import androidx.camera.core.CameraSelector.DEFAULT_BACK_CAMERA 6 7 import androidx.camera.core.CameraSelector.DEFAULT_FRONT_CAMERA 7 8 import androidx.camera.core.ExperimentalGetImage 8 9 import androidx.camera.view.PreviewView 10 + import androidx.compose.foundation.Canvas 9 11 import androidx.compose.foundation.Image 10 12 import androidx.compose.foundation.layout.Box 11 13 import androidx.compose.foundation.layout.fillMaxSize ··· 20 22 import androidx.compose.runtime.remember 21 23 import androidx.compose.runtime.setValue 22 24 import androidx.compose.ui.Modifier 25 + import androidx.compose.ui.draw.clipToBounds 23 26 import androidx.compose.ui.draw.scale 27 + import androidx.compose.ui.geometry.Offset 28 + import androidx.compose.ui.geometry.Rect 29 + import androidx.compose.ui.geometry.Size 30 + import androidx.compose.ui.graphics.Color 24 31 import androidx.compose.ui.graphics.ImageBitmap 25 32 import androidx.compose.ui.graphics.asAndroidBitmap 26 33 import androidx.compose.ui.graphics.asImageBitmap ··· 46 53 import androidx.camera.core.ImageProxy 47 54 import androidx.compose.runtime.DisposableEffect 48 55 import androidx.compose.runtime.rememberCoroutineScope 49 - import androidx.compose.ui.geometry.Rect 56 + import androidx.compose.ui.graphics.drawscope.Stroke 50 57 import com.google.mlkit.vision.pose.PoseDetector 51 58 import com.performancecoachlab.posedetection.custom.CustomObjectRespository 52 59 import com.performancecoachlab.posedetection.custom.ObjectModel ··· 123 130 onRecordToggled: (Boolean) -> Unit, 124 131 onVideoSaved: (String, String) -> Unit, 125 132 ) { 126 - var bitmap by remember { mutableStateOf<ImageBitmap?>(null) } 127 - val options = 128 - PoseDetectorOptions.Builder().setDetectorMode(PoseDetectorOptions.STREAM_MODE).build() 133 + // Replace per-frame UI bitmap with lightweight analysis state. 134 + var analysisSize by remember { mutableStateOf<Size?>(null) } 135 + var latestSkeleton by remember { mutableStateOf<Skeleton?>(null) } 136 + var latestObjects by remember { mutableStateOf<List<AnalysisObject>>(emptyList()) } 137 + 138 + val options = PoseDetectorOptions.Builder() 139 + .setDetectorMode(PoseDetectorOptions.STREAM_MODE) 140 + .build() 129 141 val poseDetector = PoseDetection.getClient(options) 142 + 130 143 val context = LocalContext.current 131 144 val lifecycleOwner: LifecycleOwner = LocalLifecycleOwner.current 132 145 val previewView: PreviewView = remember { PreviewView(context) } 133 146 val executor = remember { Executors.newSingleThreadExecutor() } 134 147 val scope = rememberCoroutineScope() 148 + 135 149 var focus by remember { mutableStateOf(focusArea) } 136 150 var objectDetector by remember { mutableStateOf(objectModel) } 137 151 var currentDetectMode by remember { mutableStateOf(detectMode) } ··· 150 164 // Multi-recording state - map of recording ID to recording slot 151 165 var activeRecordings by remember { mutableStateOf<Map<String, RecordingSlot>>(emptyMap()) } 152 166 167 + // Restored helpers (were accidentally removed during refactor) 153 168 suspend fun startRecording(id: String, width: Int, height: Int) { 154 169 val outputPath = File(context.cacheDir, "camera_recording_${id}_${System.currentTimeMillis()}.mp4").absolutePath 155 170 val slot = RecordingSlot( ··· 166 181 try { 167 182 val path = builder.finalize() 168 183 onVideoSaved(id, path) 169 - } catch (e: Exception) { 170 - //println("Error finalizing recording $id: ${e.message}") 184 + } catch (_: Exception) { 185 + // ignore finalize errors 171 186 } 172 187 } 173 188 activeRecordings = activeRecordings - id 174 189 } 175 190 } 176 191 192 + // Recording no longer depends on a continuously updated UI bitmap. 177 193 fun addFrameToActiveRecordings(frame: ImageBitmap, timestampMs: Long) { 178 194 if (activeRecordings.isNotEmpty()) { 179 195 scope.launch { ··· 208 224 val currentIds = activeRecordings.keys 209 225 recordingId?.also { id -> 210 226 currentIds.forEach { 211 - if(!it.contentEquals(id)){ 227 + if (!it.contentEquals(id)) { 212 228 stopRecording(it) 213 229 } 214 230 } 215 - if(!currentIds.contains(id)){ 216 - if (bitmap != null) { 217 - startRecording(id, bitmap!!.width, bitmap!!.height) 231 + if (!currentIds.contains(id)) { 232 + val size = analysisSize 233 + if (size != null) { 234 + startRecording(id, size.width.toInt(), size.height.toInt()) 218 235 } else { 219 236 // If no frame yet, create empty slot that will be initialized on first frame 220 237 val outputPath = File(context.cacheDir, "camera_recording_${id}_${System.currentTimeMillis()}.mp4").absolutePath ··· 226 243 activeRecordings = activeRecordings + (id to slot) 227 244 } 228 245 } 229 - }?: run { 230 - // No recordingId - stop all 231 - currentIds.forEach { 232 - stopRecording(it) 233 - } 246 + } ?: run { 247 + currentIds.forEach { stopRecording(it) } 234 248 } 235 - 236 - 237 249 } 238 250 239 251 val gate = remember { FrameGate() } ··· 267 279 val area = focus 268 280 val poseClient = if (currentDetectMode.doPose()) poseDetector else null 269 281 val objectClient = if (currentDetectMode.doObject()) objectDetector?.getDetector() else null 270 - imageProxy.process( 271 - objectClient, poseClient, timestamp, area 272 - ) { analysisResult, _bitmap -> 282 + val rotationDegrees = imageProxy.imageInfo.rotationDegrees 283 + 284 + imageProxy.process(objectClient, poseClient, timestamp, area) { analysisResult, frameBitmap -> 273 285 try { 286 + // Keep repositories as before. 274 287 customObjectRepository.updateCustomObject(analysisResult.objects) 275 - analysisResult.skeleton?.let { skel -> 276 - skeletonRepository.updateSkeleton(skel) 288 + analysisResult.skeleton?.let { skeletonRepository.updateSkeleton(it) } 289 + 290 + // Update UI overlay state. 291 + latestSkeleton = if (drawSkeleton) analysisResult.skeleton else null 292 + latestObjects = analysisResult.objects 293 + analysisSize = androidx.compose.ui.geometry.Size( 294 + width = frameBitmap.width.toFloat(), 295 + height = frameBitmap.height.toFloat() 296 + ) 297 + 298 + controller?.setRequestDataProvider { 299 + CameraViewData( 300 + width = frameBitmap.width.toFloat(), 301 + height = frameBitmap.height.toFloat(), 302 + rotation = when (rotationDegrees) { 303 + 0 -> SensorRotation.ROTATION_0 304 + 90 -> SensorRotation.ROTATION_90 305 + 180 -> SensorRotation.ROTATION_180 306 + 270 -> SensorRotation.ROTATION_270 307 + else -> SensorRotation.ROTATION_0 308 + } 309 + ) 277 310 } 278 - bitmap = _bitmap.asImageBitmap().let { inbmp -> 279 - controller?.setRequestDataProvider { 280 - CameraViewData( 281 - width = inbmp.width.toFloat(), 282 - height = inbmp.height.toFloat(), 283 - rotation = when (imageProxy.imageInfo.rotationDegrees) { 284 - 0 -> SensorRotation.ROTATION_0 285 - 90 -> SensorRotation.ROTATION_90 286 - 180 -> SensorRotation.ROTATION_180 287 - 270 -> SensorRotation.ROTATION_270 288 - else -> SensorRotation.ROTATION_0 289 - } 290 - ) 291 - } 311 + 312 + // Only convert to ImageBitmap when we're actually recording. 313 + if (activeRecordings.isNotEmpty()) { 314 + val inbmp = frameBitmap.asImageBitmap() 292 315 addFrameToActiveRecordings(inbmp, timestamp) 293 - inbmp.drawResults( 294 - if (drawSkeleton) analysisResult.skeleton else null, 295 - drawObjects?.invoke(analysisResult.objects) ?: emptyList() 296 - ) 297 316 } 298 317 } finally { 299 318 gate.exit() ··· 314 333 Logger.d { "CameraX bound cameraId=$cameraId frontCamera=$frontCamera" } 315 334 } 316 335 317 - previewView.scaleType = PreviewView.ScaleType.FIT_CENTER 336 + // IMPORTANT: keep preview scale type. 337 + previewView.scaleType = PreviewView.ScaleType.FILL_CENTER 318 338 } 339 + 319 340 Box( 320 - modifier = modifier 341 + modifier = modifier.clipToBounds() 321 342 ) { 322 - Box( 343 + AndroidView( 344 + factory = { previewView }, 323 345 modifier = Modifier 324 - .height(100.dp) 325 - .width(100.dp) 326 - .padding(20.dp) 346 + .fillMaxSize() 347 + .clipToBounds() 348 + ) 349 + 350 + val overlaySize = analysisSize 351 + 352 + Canvas( 353 + modifier = Modifier 354 + .fillMaxSize() 355 + .clipToBounds() 356 + .scale(if (frontCamera) -1f else 1f, 1f) 327 357 ) { 328 - AndroidView( 329 - factory = { previewView }, modifier = Modifier.fillMaxSize() 330 - ) 358 + val frameW = overlaySize?.width 359 + val frameH = overlaySize?.height 360 + if (frameW == null || frameH == null) return@Canvas 331 361 332 - } 333 - Surface(modifier = Modifier.fillMaxSize()) { 334 - bitmap?.also { 335 - Image( 336 - it, 337 - contentDescription = "video feed", 338 - modifier = Modifier 339 - .fillMaxSize() 340 - .scale(if (frontCamera) -1f else 1f, 1f), 341 - contentScale = ContentScale.Crop 342 - ) 362 + val viewW = size.width.coerceAtLeast(1f) 363 + val viewH = size.height.coerceAtLeast(1f) 364 + 365 + // Manual center-crop mapping matching PreviewView.ScaleType.FILL_CENTER. 366 + val scale = maxOf(viewW / frameW, viewH / frameH) 367 + val scaledW = frameW * scale 368 + val scaledH = frameH * scale 369 + 370 + // Convert from frame coords to view coords by first scaling, then centering. 371 + // dx/dy are negative when the scaled image is larger than the view (crop). 372 + val dx = (viewW - scaledW) / 2f 373 + val dy = (viewH - scaledH) / 2f 374 + 375 + fun mapX(x: Float) = dx + x * scale 376 + fun mapY(y: Float) = dy + y * scale 377 + 378 + // Objects 379 + val drawable = drawObjects?.invoke(latestObjects) ?: emptyList() 380 + drawable.forEach { d -> 381 + val bb = d.obj.boundingBox 382 + val tl = Offset(mapX(bb.left), mapY(bb.top)) 383 + val br = Offset(mapX(bb.right), mapY(bb.bottom)) 384 + 385 + val left = tl.x 386 + val top = tl.y 387 + val right = br.x 388 + val bottom = br.y 389 + 390 + val w = (right - left).coerceAtLeast(1f) 391 + val h = (bottom - top).coerceAtLeast(1f) 392 + 393 + when (d.shape) { 394 + DrawableShape.RECTANGLE -> drawRect( 395 + color = d.colour, 396 + topLeft = Offset(left, top), 397 + size = Size(w, h), 398 + style = if (d.style is androidx.compose.ui.graphics.drawscope.Fill) androidx.compose.ui.graphics.drawscope.Fill else Stroke((d.style as? Stroke)?.width ?: 3f) 399 + ) 400 + 401 + DrawableShape.OVAL -> drawOval( 402 + color = d.colour, 403 + topLeft = Offset(left, top), 404 + size = Size(w, h), 405 + style = if (d.style is androidx.compose.ui.graphics.drawscope.Fill) androidx.compose.ui.graphics.drawscope.Fill else Stroke((d.style as? Stroke)?.width ?: 3f) 406 + ) 407 + 408 + DrawableShape.LABEL -> { 409 + // no-op 410 + } 411 + } 412 + } 413 + 414 + // Skeleton 415 + latestSkeleton?.let { skel -> 416 + val joints = skel.joints() 417 + val bones = skel.bones() 418 + 419 + bones.forEach { (a, b) -> 420 + val start = Offset(mapX(a.x), mapY(a.y)) 421 + val end = Offset(mapX(b.x), mapY(b.y)) 422 + drawLine( 423 + color = Color.White, 424 + start = start, 425 + end = end, 426 + strokeWidth = 4f 427 + ) 428 + } 429 + joints.forEach { j -> 430 + val c = Offset(mapX(j.x), mapY(j.y)) 431 + drawCircle( 432 + color = Color.Blue, 433 + radius = 6f, 434 + center = c 435 + ) 436 + } 343 437 } 344 438 } 345 439 } 440 + 441 + // ...existing code... 346 442 } 347 443 348 444
+101 -19
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.android.kt
··· 31 31 import kotlin.math.min 32 32 import kotlin.run 33 33 import androidx.core.graphics.createBitmap 34 + import java.nio.ByteBuffer 34 35 35 36 actual enum class PlatformType { 36 37 ANDROID, IOS; ··· 101 102 } 102 103 } 103 104 105 + private object RgbaBufferPool { 106 + private var packed: ByteBuffer? = null 107 + private var packedCapacity: Int = 0 108 + 109 + fun packedBuffer(requiredBytes: Int): ByteBuffer { 110 + val buf = packed 111 + return if (buf != null && packedCapacity >= requiredBytes) { 112 + buf.clear() 113 + buf 114 + } else { 115 + ByteBuffer.allocateDirect(requiredBytes).also { 116 + packed = it 117 + packedCapacity = requiredBytes 118 + } 119 + } 120 + } 121 + } 122 + 104 123 private fun Bitmap.rotateToNew(degrees: Int): Bitmap { 105 124 if (degrees % 360 == 0) return this 106 125 val m = Matrix().apply { postRotate(degrees.toFloat()) } 107 126 return Bitmap.createBitmap(this, 0, 0, width, height, m, true) 108 127 } 109 128 129 + private fun Bitmap.rotateIntoPooled(degrees: Int): Bitmap { 130 + if (degrees % 360 == 0) return this 131 + val outW = if (degrees % 180 == 0) width else height 132 + val outH = if (degrees % 180 == 0) height else width 133 + 134 + val out = AnalysisBitmapPool.obtain(outW, outH, this.config ?: Bitmap.Config.ARGB_8888) 135 + val c = Canvas(out) 136 + val m = Matrix().apply { 137 + postRotate(degrees.toFloat()) 138 + // Translate back into view. 139 + when (degrees % 360) { 140 + 90 -> postTranslate(outW.toFloat(), 0f) 141 + 180 -> postTranslate(outW.toFloat(), outH.toFloat()) 142 + 270 -> postTranslate(0f, outH.toFloat()) 143 + } 144 + } 145 + c.drawBitmap(this, m, Paint(Paint.FILTER_BITMAP_FLAG)) 146 + return out 147 + } 148 + 110 149 private fun resizeInto(src: Bitmap, dst: Bitmap) { 111 150 val c = Canvas(dst) 112 151 val paint = Paint(Paint.FILTER_BITMAP_FLAG) 113 152 c.drawBitmap(src, null, android.graphics.Rect(0, 0, dst.width, dst.height), paint) 114 153 } 115 154 155 + /** 156 + * Convert an RGBA_8888 ImageProxy (from ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888) 157 + * into a Bitmap. Reuses buffers/bitmaps to keep GC pressure low. 158 + */ 159 + private fun ImageProxy.toRgbaBitmapPooled(): Bitmap { 160 + val plane = planes.firstOrNull() ?: error("ImageProxy has no planes") 161 + val buffer: ByteBuffer = plane.buffer 162 + buffer.rewind() 163 + 164 + val rowStride = plane.rowStride 165 + val pixelStride = plane.pixelStride 166 + // For OUTPUT_IMAGE_FORMAT_RGBA_8888 this should be 4. 167 + if (pixelStride != 4) { 168 + // Fallback: still attempt a direct copy into a fresh bitmap. 169 + return Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888).also { it.copyPixelsFromBuffer(buffer) } 170 + } 171 + 172 + val bmp = AnalysisBitmapPool.obtain(width, height, Bitmap.Config.ARGB_8888) 173 + 174 + if (rowStride == width * 4) { 175 + bmp.copyPixelsFromBuffer(buffer) 176 + return bmp 177 + } 178 + 179 + val required = width * height * 4 180 + val packed = RgbaBufferPool.packedBuffer(required) 181 + 182 + val rowBytes = width * 4 183 + val row = ByteArray(rowBytes) 184 + val skip = ByteArray(rowStride - rowBytes) 185 + for (y in 0 until height) { 186 + buffer.get(row) 187 + packed.put(row) 188 + if (y < height - 1) buffer.get(skip) 189 + } 190 + packed.flip() 191 + bmp.copyPixelsFromBuffer(packed) 192 + return bmp 193 + } 194 + 116 195 @OptIn(ExperimentalGetImage::class) 117 196 fun ImageProxy.process( 118 197 objectDetector: Interpreter?, ··· 130 209 val rotationDegrees = imageInfo.rotationDegrees 131 210 132 211 // Convert to bitmap ASAP so we can close the ImageProxy immediately. 133 - val analysisBitmap: Bitmap = toBitmap().rotateToNew(rotationDegrees) 212 + val baseBitmap: Bitmap = toRgbaBitmapPooled() 134 213 close() 214 + 215 + // Rotate into a pooled bitmap if needed. 216 + val analysisBitmap: Bitmap = baseBitmap.rotateIntoPooled(rotationDegrees) 135 217 136 218 // 2) MLKit image must match analysisBitmap coordinate space. Rotation is now 0. 137 219 val mlKitImage: InputImage? = poseDetector?.let { ··· 176 258 onComplete = onComplete 177 259 ) 178 260 } 179 - 180 261 private fun process( 181 262 tensorImage: TensorImage?, 182 263 mlKitImage: InputImage?, ··· 256 337 } 257 338 } else emptyList() 258 339 259 - var skeleton: Skeleton? = null 260 - val poseDetectionTask = if (poseDetector != null && mlKitImage != null) { 261 - poseDetector.process(mlKitImage) 262 - .addOnSuccessListener { pose -> 263 - skeleton = skeleton(pose, timestamp, width, height) 264 - } 265 - .addOnFailureListener { } 340 + val skeleton: Skeleton? = if (poseDetector != null && mlKitImage != null) { 341 + runCatching { 342 + val pose = Tasks.await(poseDetector.process(mlKitImage)) 343 + val landmarks = pose.allPoseLandmarks.size 344 + Logger.d { "MLKit pose landmarks=$landmarks" } 345 + skeleton(pose, timestamp, width, height) 346 + }.getOrNull() 266 347 } else null 267 348 268 - Tasks.whenAllComplete(listOfNotNull(poseDetectionTask)).addOnCompleteListener { 269 - onComplete( 270 - AnalysisResult( 271 - skeleton = skeleton, 272 - objects = objectsDetected 273 - ), 274 - bitmap 275 - ) 276 - } 349 + onComplete( 350 + AnalysisResult( 351 + skeleton = skeleton, 352 + objects = objectsDetected 353 + ), 354 + bitmap 355 + ) 277 356 } 278 357 279 358 private fun Rect?.toGraphicsRect(width: Int, height: Int):android.graphics.Rect { ··· 451 530 val cnf: Float, 452 531 val cls: Int, 453 532 val clsName: String 454 - ) 533 + ) 534 + 535 + 536 +
+29 -11
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
··· 4 4 import androidx.compose.foundation.Image 5 5 import androidx.compose.foundation.layout.Box 6 6 import androidx.compose.foundation.layout.Column 7 + import androidx.compose.foundation.layout.Row 8 + import androidx.compose.foundation.layout.Spacer 7 9 import androidx.compose.foundation.layout.WindowInsets 8 10 import androidx.compose.foundation.layout.fillMaxSize 11 + import androidx.compose.foundation.layout.fillMaxWidth 9 12 import androidx.compose.foundation.layout.imePadding 13 + import androidx.compose.foundation.layout.navigationBarsPadding 10 14 import androidx.compose.foundation.layout.padding 11 15 import androidx.compose.foundation.layout.safeDrawing 12 16 import androidx.compose.foundation.layout.windowInsetsPadding 17 + import androidx.compose.foundation.layout.wrapContentHeight 13 18 import androidx.compose.material3.Button 14 19 import androidx.compose.ui.graphics.drawscope.Stroke 15 20 import androidx.compose.material3.CircularProgressIndicator ··· 81 86 internal fun App() = AppTheme { 82 87 var selectedTabIndex by remember { mutableStateOf(0) } 83 88 val tabs = listOf("Camera Feed", "Recorded Video") 89 + 90 + // Apply safe-area insets once to the whole screen. 84 91 Column( 85 92 modifier = Modifier 86 - .fillMaxSize() 87 - .windowInsetsPadding(WindowInsets.safeDrawing) 93 + .fillMaxSize() 94 + .windowInsetsPadding(WindowInsets.safeDrawing) 88 95 ) { 89 - TabRow(selectedTabIndex = selectedTabIndex) { 96 + // Tab row should be only as tall as needed. 97 + TabRow(selectedTabIndex = selectedTabIndex, modifier = Modifier.fillMaxWidth()) { 90 98 tabs.forEachIndexed { index, title -> 91 99 Tab( 92 100 selected = selectedTabIndex == index, 93 101 onClick = { selectedTabIndex = index }, 94 - text = { Text(title) }) 102 + text = { Text(title) } 103 + ) 95 104 } 96 105 } 97 - when (selectedTabIndex) { 98 - 0 -> CameraSample() 99 - 1 -> RecordedSample() 106 + 107 + // Content always gets the remaining space. 108 + Box( 109 + modifier = Modifier 110 + .weight(1f) 111 + .fillMaxWidth() 112 + ) { 113 + when (selectedTabIndex) { 114 + 0 -> CameraSample() 115 + 1 -> RecordedSample() 116 + } 100 117 } 101 118 } 102 119 } ··· 357 374 CameraView( 358 375 skeletonRepository = skeletonRepository, 359 376 customObjectRepository = customObjectRespository, 360 - detectMode = DetectMode.OBJECT, 377 + detectMode = DetectMode.BOTH, 361 378 drawSkeleton = true, 362 379 drawObjects = { obj -> 363 380 obj.flatMap { ··· 370 387 ), 371 388 DrawableObject( 372 389 obj = it, 373 - shape = DrawableShape.OVAL, 390 + shape = DrawableShape.RECTANGLE, 374 391 colour = Color.Green, 375 392 style = Stroke(it.boundingBox.width * 0.1f) 376 393 ) ··· 379 396 }, 380 397 objectModel = generalModel, 381 398 modifier = Modifier.weight(1f), 382 - focusArea = Rect(0f,0f,0.1f,1f), 399 + // focusArea = Rect(0f,0f,0.1f,1f), 400 + focusArea = null, 383 401 frontCamera = frontCamera, 384 402 recordingId = recordingId, 385 403 controller = controller, 386 - onVideoSaved = {id,url -> path = url }, 404 + onVideoSaved = { id, url -> path = url }, 387 405 ) 388 406 } 389 407 Button(