Merge remote-tracking branch 'refs/remotes/origin/release-4.10.0' · nateholland.bsky.social/PoseDetection@426cfea

+1

.gitignore

··· 16 16 *.gpg 17 17 *yarn.lock 18 18 /gradle.properties 19 + *.hprof

+1 -1

posedetection/build.gradle.kts

··· 4 4 5 5 mavenPublishing { 6 6 publishToMavenCentral(SonatypeHost.CENTRAL_PORTAL) 7 - coordinates("com.performancecoachlab.posedetection", "posedetection-compose", "4.9.2") 7 + coordinates("com.performancecoachlab.posedetection", "posedetection-compose", "4.10.0") 8 8 9 9 pom { 10 10 name.set("Pose Detection")

+19 -4

posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt

··· 134 134 val poseIntervalMs = 33L 135 135 val lastObjectRunAtMs = remember { AtomicLong(0L) } 136 136 val lastPoseRunAtMs = remember { AtomicLong(0L) } 137 + val analysisFrameCounter = remember { AtomicLong(0L) } 137 138 138 139 val options = PoseDetectorOptions.Builder() 139 140 .setDetectorMode(PoseDetectorOptions.STREAM_MODE) ··· 150 151 var currentDetectMode by remember { mutableStateOf(detectMode) } 151 152 152 153 val textMeasurer = rememberTextMeasurer() 153 - LaunchedEffect(detectMode) { currentDetectMode = detectMode } 154 + LaunchedEffect(detectMode) { 155 + currentDetectMode = detectMode 156 + if (!detectMode.doPose()) latestSkeleton = null 157 + if (!detectMode.doObject()) latestObjects = emptyList() 158 + } 154 159 155 160 // Update focus when focusArea changes 156 161 LaunchedEffect(focusArea) { ··· 291 296 return@setAnalyzer 292 297 } 293 298 294 - val now = System.currentTimeMillis() 299 + // Convert sensor timestamp to epoch time. 300 + // sensor is REALTIME ns, convert to epoch ms via boot offset. 301 + val sensorMs = imageProxy.imageInfo.timestamp / 1_000_000L 302 + val bootEpochOffset = System.currentTimeMillis() - android.os.SystemClock.elapsedRealtime() 303 + val now = sensorMs + bootEpochOffset 295 304 val area = focus 296 305 306 + // In BOTH mode, stagger pose and object on alternate frames 307 + // to reduce per-frame processing time and improve tracking. 308 + val frameNum = analysisFrameCounter.incrementAndGet() 309 + val isBothMode = currentDetectMode == DetectMode.BOTH 297 310 val shouldRunObject = currentDetectMode.doObject() && 298 - (now - lastObjectRunAtMs.get() >= objectIntervalMs) 311 + (now - lastObjectRunAtMs.get() >= objectIntervalMs) && 312 + (!isBothMode || frameNum % 2 == 0L) 299 313 val shouldRunPose = currentDetectMode.doPose() && 300 - (now - lastPoseRunAtMs.get() >= poseIntervalMs) 314 + (now - lastPoseRunAtMs.get() >= poseIntervalMs) && 315 + (!isBothMode || frameNum % 2 != 0L) 301 316 302 317 // If neither detector is scheduled to run, just close quickly and reuse last results. 303 318 if (!shouldRunObject && !shouldRunPose) {

+13 -4

posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.android.kt

··· 326 326 val outputShape = objectDetector.modelInfo.outputShape 327 327 val output = TensorBuffer.createFixedSize(outputShape, DataType.FLOAT32) 328 328 329 - objectDetector.interpreter.run(tensorImage.buffer, output.buffer) 329 + try { 330 + objectDetector.interpreter.run(tensorImage.buffer, output.buffer) 331 + } catch (e: Exception) { 332 + Logger.e(e) { "TFLite interpreter.run failed" } 333 + val skeleton = poseFuture?.get() 334 + onComplete(AnalysisResult(skeleton, emptyList(), timestamp), bitmap) 335 + return 336 + } 330 337 331 338 val array = output.floatArray 332 339 if (outputShape.size != 3) emptyList() else { ··· 353 360 else -> { 354 361 val skeleton = poseFuture?.get() 355 362 return onComplete( 356 - AnalysisResult(skeleton = skeleton, objects = emptyList()), 363 + AnalysisResult(skeleton = skeleton, objects = emptyList(), timestamp = timestamp), 357 364 bitmap 358 365 ) 359 366 } ··· 405 412 frameSize = FrameSize( 406 413 width = width.absoluteValue, 407 414 height = height.absoluteValue 408 - ) 415 + ), 416 + timestamp = timestamp 409 417 ) 410 418 } else null 411 419 } ··· 418 426 onComplete( 419 427 AnalysisResult( 420 428 skeleton = skeleton, 421 - objects = objectsDetected 429 + objects = objectsDetected, 430 + timestamp = timestamp 422 431 ), 423 432 bitmap 424 433 )

+7 -2

posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/encoding/VideoBuilder.android.kt

··· 111 111 112 112 // NV12 format: y plane followed by interleaved uv plane 113 113 val ySize = width * height 114 - val uvSize = ySize / 2 114 + val uvSize = ((width + 1) / 2) * ((height + 1) / 2) * 2 115 115 val yuvSize = ySize + uvSize 116 116 val yuv = ByteArray(yuvSize) 117 117 ··· 187 187 muxer = MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) 188 188 initialised = true 189 189 } 190 - val bitmap = frame.asAndroidBitmap() 190 + val raw = frame.asAndroidBitmap() 191 + val bitmap = if (raw.width != width || raw.height != height) { 192 + Bitmap.createScaledBitmap(raw, width, height, true) 193 + } else { 194 + raw 195 + } 191 196 val yuvData = convertBitmapToYuv420(bitmap) 192 197 val inputBufferIndex = try { 193 198 encoder.dequeueInputBuffer(timeoutUs)

+67 -15

posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt

··· 11 11 import androidx.core.net.toUri 12 12 import co.touchlab.kermit.Logger 13 13 import kotlinx.coroutines.Dispatchers 14 + import kotlinx.coroutines.flow.Flow 15 + import kotlinx.coroutines.flow.flow 16 + import kotlinx.coroutines.flow.flowOn 14 17 import kotlinx.coroutines.sync.Mutex 15 18 import kotlinx.coroutines.sync.withLock 16 19 import kotlinx.coroutines.withContext ··· 81 84 return lastBitmap 82 85 } 83 86 84 - val timeoutUs = 10_000L 85 87 val info = MediaCodec.BufferInfo() 86 88 87 89 try { 88 90 while (!outputEos) { 89 - // Feed input packets. 91 + // Feed as many input packets as possible to keep the decoder pipeline full. 90 92 if (!inputEos) { 91 - val inIdx = decoder.dequeueInputBuffer(0) 92 - if (inIdx >= 0) { 93 + while (true) { 94 + val inIdx = decoder.dequeueInputBuffer(10_000L) 95 + if (inIdx < 0) break 93 96 val buf = decoder.getInputBuffer(inIdx)!! 94 97 val size = ext.readSampleData(buf, 0) 95 98 if (size < 0) { ··· 97 100 inIdx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM 98 101 ) 99 102 inputEos = true 103 + break 100 104 } else { 101 105 decoder.queueInputBuffer(inIdx, 0, size, ext.sampleTime, 0) 102 106 ext.advance() ··· 105 109 } 106 110 107 111 // Drain output. 108 - val outIdx = decoder.dequeueOutputBuffer(info, timeoutUs) 112 + val outIdx = decoder.dequeueOutputBuffer(info, 10_000L) 109 113 if (outIdx >= 0) { 110 114 val ptsMs = info.presentationTimeUs / 1000L 111 115 ··· 162 166 return Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true) 163 167 } 164 168 169 + /** 170 + * Fast YUV Image to ARGB Bitmap conversion using bulk buffer copies 171 + * and fixed-point integer arithmetic (no floating-point per pixel). 172 + */ 165 173 private fun yuvImageToBitmap(image: Image): Bitmap { 166 174 val w = image.width 167 175 val h = image.height ··· 169 177 val uPlane = image.planes[1] 170 178 val vPlane = image.planes[2] 171 179 180 + val yRowStride = yPlane.rowStride 181 + val uvRowStride = uPlane.rowStride 182 + val uvPixelStride = uPlane.pixelStride 183 + 184 + // Bulk-copy ByteBuffers to ByteArrays to avoid per-pixel ByteBuffer.get() overhead. 172 185 val yBuf = yPlane.buffer 173 186 val uBuf = uPlane.buffer 174 187 val vBuf = vPlane.buffer 175 188 176 - val yRowStride = yPlane.rowStride 177 - val uvRowStride = uPlane.rowStride 178 - val uvPixelStride = uPlane.pixelStride 189 + val yBytes = ByteArray(yBuf.remaining()) 190 + yBuf.get(yBytes) 191 + val uBytes = ByteArray(uBuf.remaining()) 192 + uBuf.get(uBytes) 193 + val vBytes = ByteArray(vBuf.remaining()) 194 + vBuf.get(vBytes) 179 195 180 196 val argb = IntArray(w * h) 181 197 182 198 for (j in 0 until h) { 199 + val yRowOffset = j * yRowStride 200 + val uvRowOffset = (j shr 1) * uvRowStride 183 201 for (i in 0 until w) { 184 - val y = (yBuf.get(j * yRowStride + i).toInt() and 0xFF) 185 - val uvIdx = (j / 2) * uvRowStride + (i / 2) * uvPixelStride 186 - val u = (uBuf.get(uvIdx).toInt() and 0xFF) - 128 187 - val v = (vBuf.get(uvIdx).toInt() and 0xFF) - 128 202 + val y = yBytes[yRowOffset + i].toInt() and 0xFF 203 + val uvIdx = uvRowOffset + (i shr 1) * uvPixelStride 204 + val u = (uBytes[uvIdx].toInt() and 0xFF) - 128 205 + val v = (vBytes[uvIdx].toInt() and 0xFF) - 128 188 206 189 - val r = (y + 1.370705 * v).toInt().coerceIn(0, 255) 190 - val g = (y - 0.337633 * u - 0.698001 * v).toInt().coerceIn(0, 255) 191 - val b = (y + 1.732446 * u).toInt().coerceIn(0, 255) 207 + // Fixed-point: multiply by scaled constant, shift right by 10. 208 + // 1.402 * 1024 = 1436, 0.344 * 1024 = 352, 0.714 * 1024 = 731, 1.772 * 1024 = 1815 209 + var r = y + ((v * 1436) shr 10) 210 + var g = y - ((u * 352 + v * 731) shr 10) 211 + var b = y + ((u * 1815) shr 10) 212 + if (r < 0) r = 0 else if (r > 255) r = 255 213 + if (g < 0) g = 0 else if (g > 255) g = 255 214 + if (b < 0) b = 0 else if (b > 255) b = 255 192 215 193 216 argb[j * w + i] = (0xFF shl 24) or (r shl 16) or (g shl 8) or b 194 217 } ··· 229 252 } 230 253 } 231 254 } 255 + 256 + /** 257 + * Batch sequential frame extraction as a Flow. Opens a dedicated decoder, 258 + * decodes frames in order, and emits each as it's ready. More efficient than 259 + * repeated [extractFrame] calls because there's no per-call mutex/coroutine 260 + * overhead and the decoder runs continuously. 261 + */ 262 + actual fun extractFrames( 263 + videoPath: String, frameTimestamps: List<Long>, cropRect: CropRect? 264 + ): Flow<InputFrame> = flow { 265 + val decoder = VideoFrameDecoder(videoPath) 266 + try { 267 + for (ts in frameTimestamps) { 268 + val bitmap = decoder.decodeUpTo(ts) 269 + if (bitmap != null) { 270 + val output = if (cropRect != null) { 271 + val cx = cropRect.x.coerceIn(0, bitmap.width - 1) 272 + val cy = cropRect.y.coerceIn(0, bitmap.height - 1) 273 + val cw = cropRect.width.coerceAtMost(bitmap.width - cx) 274 + val ch = cropRect.height.coerceAtMost(bitmap.height - cy) 275 + Bitmap.createBitmap(bitmap, cx, cy, cw, ch) 276 + } else bitmap 277 + emit(InputFrame(bitmap = output, timestamp = ts)) 278 + } 279 + } 280 + } finally { 281 + decoder.release() 282 + } 283 + }.flowOn(Dispatchers.IO) 232 284 233 285 actual suspend fun listVideoFrameTimestamps( 234 286 videoPath: String,

+32 -13

posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.kt

··· 234 234 val drawScope = CanvasDrawScope() 235 235 val size = Size(it.width.toFloat(), it.height.toFloat()) 236 236 237 - // Calculate scaling factor based on skeleton size 237 + // Compute scale factors to map detection coordinates to target bitmap. 238 + // Use independent X/Y scaling — handles the case where the skeleton's 239 + // coordinate space has a different aspect ratio than the target frame 240 + // (e.g., iOS portrait analysis vs landscape extracted video). 241 + val skelScaleX = if (skeleton != null && skeleton.width > 0f) 242 + it.width.toFloat() / skeleton.width else 1f 243 + val skelScaleY = if (skeleton != null && skeleton.height > 0f) 244 + it.height.toFloat() / skeleton.height else 1f 245 + val skelOffsetX = 0f 246 + val skelOffsetY = 0f 247 + 248 + // Calculate scaling factor based on skeleton size (after mapping to target) 238 249 val skeletonSize = skeleton?.joints()?.let { joints -> 239 - val minX = joints.minOfOrNull { joint -> joint.x } ?: 0f 240 - val maxX = joints.maxOfOrNull { joint -> joint.x } ?: 0f 241 - val minY = joints.minOfOrNull { joint -> joint.y } ?: 0f 242 - val maxY = joints.maxOfOrNull { joint -> joint.y } ?: 0f 250 + val minX = joints.minOfOrNull { joint -> joint.x * skelScaleX } ?: 0f 251 + val maxX = joints.maxOfOrNull { joint -> joint.x * skelScaleX } ?: 0f 252 + val minY = joints.minOfOrNull { joint -> joint.y * skelScaleY } ?: 0f 253 + val maxY = joints.maxOfOrNull { joint -> joint.y * skelScaleY } ?: 0f 243 254 kotlin.math.max(maxX - minX, maxY - minY) 244 255 } ?: 1f 245 256 val minDime = kotlin.math.min(it.width, it.height).toFloat() ··· 255 266 ) { 256 267 drawImage(it) 257 268 analysisResults.objects.forEach { analysisObject -> 269 + val oScaleX = if (analysisObject.frameSize.width > 0) 270 + it.width.toFloat() / analysisObject.frameSize.width else 1f 271 + val oScaleY = if (analysisObject.frameSize.height > 0) 272 + it.height.toFloat() / analysisObject.frameSize.height else 1f 258 273 drawRect( 259 274 color = Color.Red, topLeft = androidx.compose.ui.geometry.Offset( 260 - analysisObject.boundingBox.left, analysisObject.boundingBox.top 275 + analysisObject.boundingBox.left * oScaleX, 276 + analysisObject.boundingBox.top * oScaleY 261 277 ), size = Size( 262 - analysisObject.boundingBox.width, analysisObject.boundingBox.height 278 + analysisObject.boundingBox.width * oScaleX, 279 + analysisObject.boundingBox.height * oScaleY 263 280 ), style = Stroke(scaledStrokeWidth) 264 281 ) 265 282 } ··· 278 295 bones().forEach { line -> 279 296 drawLine( 280 297 color = paintWhite.color, start = androidx.compose.ui.geometry.Offset( 281 - line.first.x, line.first.y 298 + line.first.x * skelScaleX + skelOffsetX, line.first.y * skelScaleY + skelOffsetY 282 299 ), end = androidx.compose.ui.geometry.Offset( 283 - line.second.x, line.second.y 300 + line.second.x * skelScaleX + skelOffsetX, line.second.y * skelScaleY + skelOffsetY 284 301 ), strokeWidth = paintWhite.strokeWidth, blendMode = BlendMode.Softlight 285 302 ) 286 303 drawLine( 287 304 color = paintBlue.color, start = androidx.compose.ui.geometry.Offset( 288 - line.first.x, line.first.y 305 + line.first.x * skelScaleX + skelOffsetX, line.first.y * skelScaleY + skelOffsetY 289 306 ), end = androidx.compose.ui.geometry.Offset( 290 - line.second.x, line.second.y 307 + line.second.x * skelScaleX + skelOffsetX, line.second.y * skelScaleY + skelOffsetY 291 308 ), strokeWidth = paintBlue.strokeWidth, blendMode = BlendMode.Color 292 309 ) 293 310 } 294 311 295 312 joints().forEach { joint -> 313 + val jx = joint.x * skelScaleX + skelOffsetX 314 + val jy = joint.y * skelScaleY + skelOffsetY 296 315 drawCircle( 297 316 brush = Brush.radialGradient( 298 317 colors = listOf(Color.Blue, Color.Transparent), 299 - center = androidx.compose.ui.geometry.Offset(joint.x, joint.y), 318 + center = androidx.compose.ui.geometry.Offset(jx, jy), 300 319 radius = 1.2f * scaledStrokeWidth 301 320 ), 302 321 radius = 1.2f * scaledStrokeWidth, 303 - center = androidx.compose.ui.geometry.Offset(joint.x, joint.y) 322 + center = androidx.compose.ui.geometry.Offset(jx, jy) 304 323 ) 305 324 } 306 325 }

+1 -1

posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectRespository.kt

··· 11 11 fun updateCustomObject(customObject: List<AnalysisObject>) { 12 12 _customObjectFlow.value = customObject 13 13 } 14 - } 14 + }

+4 -2

posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.kt

··· 22 22 val boundingBox: Rect, 23 23 val trackingId: Int?, 24 24 val labels: List<Label>, 25 - val frameSize: FrameSize 25 + val frameSize: FrameSize, 26 + val timestamp: Long = 0L 26 27 ) 27 28 28 29 data class FrameSize( ··· 37 38 38 39 data class AnalysisResult( 39 40 val skeleton: Skeleton?, 40 - val objects: List<AnalysisObject> 41 + val objects: List<AnalysisObject>, 42 + val timestamp: Long = skeleton?.timestamp ?: 0L 41 43 )

+24 -1

posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt

··· 1 1 package com.performancecoachlab.posedetection.recording 2 2 3 + import kotlinx.coroutines.flow.Flow 4 + 5 + /** 6 + * Pixel-coordinate crop rectangle applied after decode + rotation. 7 + * [x] and [y] are the top-left corner; [width] and [height] are the crop size. 8 + */ 9 + data class CropRect(val x: Int, val y: Int, val width: Int, val height: Int) 10 + 3 11 expect suspend fun extractFrame( 4 12 videoPath: String, frameTimestamp: Long 5 13 ): InputFrame? 6 14 15 + /** 16 + * Batch sequential frame extraction as a Flow. More efficient than repeated 17 + * [extractFrame] calls for processing many frames from the same video. 18 + * Frames are emitted in the order of [frameTimestamps] (must be ascending). 19 + * 20 + * If [cropRect] is provided, each frame is cropped to those pixel coordinates 21 + * after decode and rotation. This avoids processing full-resolution pixels 22 + * downstream. 23 + */ 24 + expect fun extractFrames( 25 + videoPath: String, 26 + frameTimestamps: List<Long>, 27 + cropRect: CropRect? = null, 28 + ): Flow<InputFrame> 29 + 7 30 expect suspend fun listVideoFrameTimestamps( 8 31 videoPath: String, 9 32 ): List<Long> 10 33 11 34 @Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") 12 - expect object VideoExtractionContext 35 + expect object VideoExtractionContext

+34

posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/skeleton/Skeleton.kt

··· 37 37 val rightKnee: Double?, 38 38 ) 39 39 40 + companion object { 41 + /** Linearly interpolate between two skeletons. [alpha] in 0..1. */ 42 + fun lerp(a: Skeleton, b: Skeleton, alpha: Float): Skeleton { 43 + fun lerpCoord( 44 + c1: SkeletonCoordinate?, c2: SkeletonCoordinate?, t: Float 45 + ): SkeletonCoordinate? { 46 + if (c1 == null) return c2 47 + if (c2 == null) return c1 48 + return SkeletonCoordinate( 49 + x = c1.x + (c2.x - c1.x) * t, 50 + y = c1.y + (c2.y - c1.y) * t 51 + ) 52 + } 53 + val t = alpha.coerceIn(0f, 1f) 54 + return Skeleton( 55 + timestamp = (a.timestamp + ((b.timestamp - a.timestamp) * t).toLong()), 56 + leftShoulder = lerpCoord(a.leftShoulder, b.leftShoulder, t), 57 + rightShoulder = lerpCoord(a.rightShoulder, b.rightShoulder, t), 58 + leftElbow = lerpCoord(a.leftElbow, b.leftElbow, t), 59 + rightElbow = lerpCoord(a.rightElbow, b.rightElbow, t), 60 + leftWrist = lerpCoord(a.leftWrist, b.leftWrist, t), 61 + rightWrist = lerpCoord(a.rightWrist, b.rightWrist, t), 62 + leftHip = lerpCoord(a.leftHip, b.leftHip, t), 63 + rightHip = lerpCoord(a.rightHip, b.rightHip, t), 64 + leftKnee = lerpCoord(a.leftKnee, b.leftKnee, t), 65 + rightKnee = lerpCoord(a.rightKnee, b.rightKnee, t), 66 + leftAnkle = lerpCoord(a.leftAnkle, b.leftAnkle, t), 67 + rightAnkle = lerpCoord(a.rightAnkle, b.rightAnkle, t), 68 + width = a.width, 69 + height = a.height, 70 + ) 71 + } 72 + } 73 + 40 74 fun bones(): List<Pair<SkeletonCoordinate, SkeletonCoordinate>> { 41 75 val lines = emptyList<Pair<SkeletonCoordinate, SkeletonCoordinate>>().toMutableList() 42 76 if (leftShoulder != null && rightShoulder != null) lines += Pair(

+4 -2

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt

··· 350 350 frameSize = FrameSize( 351 351 width = abs(width.toInt()), 352 352 height = abs(height.toInt()) 353 - ) 353 + ), 354 + timestamp = timestamp 354 355 ) 355 356 } 356 357 onObjectsProcessed(analysisObjects) ··· 533 534 frameSize = FrameSize( 534 535 width = orientedSize.width.toInt().absoluteValue, 535 536 height = orientedSize.height.toInt().absoluteValue 536 - ) 537 + ), 538 + timestamp = timestamp 537 539 ) 538 540 } 539 541

+55 -13

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/encoding/VideoBuilder.ios.kt

··· 1 1 package com.performancecoachlab.posedetection.encoding 2 2 3 + import co.touchlab.kermit.Logger 3 4 import androidx.compose.ui.graphics.ImageBitmap 4 5 import androidx.compose.ui.graphics.PixelMap 5 6 import androidx.compose.ui.graphics.toPixelMap ··· 16 17 import kotlinx.cinterop.value 17 18 import kotlinx.coroutines.Dispatchers 18 19 import kotlinx.coroutines.withContext 20 + import platform.Foundation.NSThread 19 21 import platform.AVFoundation.AVAssetWriter 20 22 import platform.AVFoundation.AVAssetWriterInput 21 23 import platform.AVFoundation.AVAssetWriterInputPixelBufferAdaptor ··· 115 117 frameCount = 0L 116 118 } 117 119 120 + private var lastTimestampMs = -1L 121 + 118 122 @OptIn(ExperimentalForeignApi::class, NativeRuntimeApi::class) 119 123 override suspend fun addFrame(frame: ImageBitmap, timestampms: Long) { 120 124 if (!started) initWriter() 121 - // Check for empty or invalid frame (skip if width or height is 0) 122 125 if (frame.width == 0 || frame.height == 0) return 126 + // Ensure strictly monotonic timestamps. 127 + val ts = if (timestampms <= lastTimestampMs) lastTimestampMs + 1 else timestampms 128 + lastTimestampMs = ts 123 129 val videoWriterInput = input!! 124 130 val pixelBufferAdaptor = adaptor!! 125 131 val pool = pixelBufferAdaptor.pixelBufferPool 126 - ?: throw IllegalStateException("Pixel buffer pool is null") 127 - val presentationTime = CMTimeMake(timestampms, 1_000) 132 + if (pool == null) { 133 + Logger.w { "VideoBuilder: pool null at frame $frameCount" } 134 + return 135 + } 136 + val presentationTime = CMTimeMake(ts, 1_000) 128 137 memScoped { 129 138 val pixelBufferPtr = alloc<CVPixelBufferRefVar>() 130 139 val status = ··· 173 182 } 174 183 CGColorSpaceRelease(colorSpace) 175 184 CVPixelBufferUnlockBaseAddress(pixelBuffer, 0u) 176 - // Wait for input to be ready 185 + // Wait for input to be ready (with timeout guard) 186 + var waitCount = 0 177 187 while (!videoWriterInput.readyForMoreMediaData) { 188 + waitCount++ 189 + if (waitCount > 10_000_000) { 190 + Logger.w { "VideoBuilder: addFrame timeout at frame $frameCount" } 191 + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0u) 192 + CVPixelBufferRelease(pixelBuffer) 193 + return 194 + } 178 195 } 179 196 pixelBufferAdaptor.appendPixelBuffer( 180 197 pixelBuffer, ··· 189 206 } 190 207 191 208 @OptIn(ExperimentalForeignApi::class) 192 - override suspend fun finalize(): String = withContext(Dispatchers.Default) { 193 - if (started) { 194 - input?.markAsFinished() 195 - writer?.finishWritingWithCompletionHandler { 196 - //println("[VideoBuilder] Finished writing video!") 209 + override suspend fun finalize(): String { 210 + if (!started) return outputPath 211 + Logger.d { "VideoBuilder: finalize() starting, frameCount=$frameCount" } 212 + val inp = input 213 + val w = writer 214 + if (inp == null || w == null) { 215 + cleanup() 216 + return outputPath 217 + } 218 + try { 219 + inp.markAsFinished() 220 + Logger.d { "VideoBuilder: input marked finished" } 221 + 222 + // finishWritingWithCompletionHandler must be called from main thread. 223 + // The completion handler fires on AVFoundation's internal queue. 224 + // Use a truly empty handler and poll w.status instead. 225 + w.finishWritingWithCompletionHandler { 226 + // Intentionally empty - do NOT reference any Kotlin objects here 227 + // as this runs on an AVFoundation internal dispatch queue 197 228 } 198 - // Wait for writing to finish 199 - while (writer?.status == AVAssetWriterStatusWriting) { 229 + Logger.d { "VideoBuilder: finishWriting called, waiting..." } 230 + var waitCount = 0 231 + while (w.status == AVAssetWriterStatusWriting && waitCount < 3000) { 232 + NSThread.sleepForTimeInterval(0.01) 233 + waitCount++ 200 234 } 201 - cleanup() 235 + Logger.d { "VideoBuilder: done, status=${w.status}, waited=${waitCount * 10}ms" } 236 + } catch (e: Exception) { 237 + Logger.e(e) { "VideoBuilder: finalize exception" } 238 + } 239 + if (w.status == platform.AVFoundation.AVAssetWriterStatusFailed) { 240 + Logger.e { "VideoBuilder: finalize FAILED: ${w.error?.localizedDescription}" } 241 + } else { 242 + Logger.d { "VideoBuilder: finalize SUCCESS, frames=$frameCount" } 202 243 } 203 - outputPath 244 + cleanup() 245 + return outputPath 204 246 } 205 247 206 248 override suspend fun cancel() = withContext(Dispatchers.Default) {

+19

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt

··· 10 10 import kotlinx.cinterop.value 11 11 import kotlinx.coroutines.Dispatchers 12 12 import kotlinx.coroutines.IO 13 + import kotlinx.coroutines.flow.Flow 14 + import kotlinx.coroutines.flow.flow 15 + import kotlinx.coroutines.flow.flowOn 13 16 import kotlinx.coroutines.withContext 14 17 import platform.AVFoundation.AVAsset 15 18 import platform.AVFoundation.AVAssetImageGenerator ··· 55 58 return@withContext null 56 59 } 57 60 } 61 + 62 + actual fun extractFrames( 63 + videoPath: String, frameTimestamps: List<Long>, cropRect: CropRect? 64 + ): Flow<InputFrame> = flow { 65 + for (ts in frameTimestamps) { 66 + val frame = extractFrame(videoPath, ts) ?: continue 67 + val output = if (cropRect != null) { 68 + frame.crop(androidx.compose.ui.geometry.Rect( 69 + cropRect.x.toFloat(), cropRect.y.toFloat(), 70 + (cropRect.x + cropRect.width).toFloat(), 71 + (cropRect.y + cropRect.height).toFloat() 72 + )) 73 + } else frame 74 + emit(output) 75 + } 76 + }.flowOn(Dispatchers.IO) 58 77 59 78 private fun NSURL.safeDescription(): String { 60 79 val scheme = this.scheme ?: "unknown"

sample/composeApp/src/androidMain/assets/yolo11n_dataset_dataset.tflite

This is a binary file and will not be displayed.

sample/composeApp/src/androidMain/assets/yolo11n_su_416.tflite

This is a binary file and will not be displayed.

+446

sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt

··· 2 2 3 3 import androidx.compose.animation.AnimatedContent 4 4 import androidx.compose.foundation.Image 5 + import androidx.compose.foundation.pager.HorizontalPager 6 + import androidx.compose.foundation.pager.rememberPagerState 5 7 import androidx.compose.foundation.layout.Arrangement 6 8 import androidx.compose.foundation.layout.Box 7 9 import androidx.compose.foundation.layout.Column ··· 9 11 import androidx.compose.foundation.layout.Spacer 10 12 import androidx.compose.foundation.layout.WindowInsets 11 13 import androidx.compose.foundation.layout.fillMaxSize 14 + import androidx.compose.ui.layout.onSizeChanged 12 15 import androidx.compose.foundation.layout.fillMaxWidth 13 16 import androidx.compose.foundation.layout.imePadding 14 17 import androidx.compose.foundation.layout.padding ··· 39 42 import androidx.compose.ui.Alignment 40 43 import androidx.compose.ui.Modifier 41 44 import androidx.compose.ui.geometry.Rect 45 + import androidx.compose.ui.geometry.Size 46 + import androidx.compose.ui.graphics.Canvas 42 47 import androidx.compose.ui.graphics.Color 43 48 import androidx.compose.ui.graphics.ImageBitmap 49 + import androidx.compose.ui.graphics.drawscope.CanvasDrawScope 44 50 import androidx.compose.ui.graphics.drawscope.Stroke 51 + import androidx.compose.ui.unit.Density 52 + import androidx.compose.ui.unit.LayoutDirection 45 53 import androidx.compose.ui.layout.ContentScale 46 54 import androidx.compose.ui.text.style.TextAlign 47 55 import androidx.compose.ui.unit.dp ··· 64 72 import com.performancecoachlab.posedetection.encoding.VideoBuilder 65 73 import com.performancecoachlab.posedetection.encoding.createVideoBuilder 66 74 import com.performancecoachlab.posedetection.permissions.PermissionProvider 75 + import com.performancecoachlab.posedetection.recording.AnalysisObject 67 76 import com.performancecoachlab.posedetection.recording.FrameAnalyser 68 77 import com.performancecoachlab.posedetection.recording.InputFrame 78 + import com.performancecoachlab.posedetection.recording.AnalysisResult 79 + import com.performancecoachlab.posedetection.recording.FrameSize 69 80 import com.performancecoachlab.posedetection.recording.extractFrame 81 + import com.performancecoachlab.posedetection.recording.extractFrames 70 82 import com.performancecoachlab.posedetection.recording.listVideoFrameTimestamps 71 83 import com.performancecoachlab.posedetection.skeleton.Pose 84 + import com.performancecoachlab.posedetection.skeleton.Skeleton 72 85 import com.performancecoachlab.posedetection.skeleton.SkeletonRepository 73 86 import io.github.vinceglb.filekit.FileKit 74 87 import io.github.vinceglb.filekit.PlatformFile ··· 79 92 import io.github.vinceglb.filekit.extension 80 93 import io.github.vinceglb.filekit.filesDir 81 94 import io.github.vinceglb.filekit.path 95 + import kotlinx.coroutines.CompletableDeferred 82 96 import kotlinx.coroutines.Job 97 + import kotlinx.coroutines.delay 83 98 import kotlinx.coroutines.launch 84 99 import kotlin.math.roundToLong 85 100 import kotlin.time.Clock ··· 376 391 CircularProgressIndicator( 377 392 modifier = Modifier.align(Alignment.Center), 378 393 ) 394 + } 395 + } 396 + } 397 + 398 + // DebugTestScreen removed. 399 + // CameraSample starts below. 400 + private object DebugTestScreenRemoved 401 + 402 + @Suppress("unused") 403 + @Composable 404 + private fun debugTestScreenRemoved() { 405 + val availableModels = discoverModels() 406 + var selectedModel by remember(availableModels) { mutableStateOf(availableModels.firstOrNull()) } 407 + val modelPath = selectedModel?.path ?: ModelPath() 408 + val generalModel = rememberObjectModel(modelPath) 409 + 410 + val skeletonRepository = remember { SkeletonRepository() } 411 + val customObjectRepository = remember { CustomObjectRespository() } 412 + val skeleton by skeletonRepository.skeletonFlow.collectAsState() 413 + val customObjects by customObjectRepository.customObjectFlow.collectAsState() 414 + var permissionGranted by remember { mutableStateOf(false) } 415 + var detectMode by remember { mutableStateOf(DetectMode.BOTH) } 416 + 417 + // Test phases: WAITING_PERMISSION -> PREVIEW -> RECORDING -> COMPARING -> DONE 418 + var phase by remember { mutableStateOf("WAITING_PERMISSION") } 419 + var recordingId: String? by remember { mutableStateOf(null) } 420 + var originalVideoPath by remember { mutableStateOf("") } 421 + var processedVideoPath: String? by remember { mutableStateOf(null) } 422 + // Comparison test data 423 + data class ComparisonResult( 424 + val image: ImageBitmap, 425 + val label: String, 426 + val frameTs: Long, // video PTS of this frame 427 + val frameElapsedMs: Long, // elapsed from video start 428 + val expectedElapsedMs: Long, // where the skeleton was expected to match 429 + val liveSkeleton: Skeleton?, 430 + val videoPath: String, 431 + val allTimestamps: List<Long>, 432 + val angleMatch: Float, 433 + val posMatch: Float 434 + ) 435 + var comparisonImages by remember { mutableStateOf<List<ComparisonResult>>(emptyList()) } 436 + var findBestResult by remember { mutableStateOf<Pair<Int, List<Pair<ImageBitmap, String>>>?>(null) } 437 + var statusText by remember { mutableStateOf("Waiting for camera permission...") } 438 + 439 + 440 + // Timing 441 + var recordTimeMs by remember { mutableStateOf(0L) } 442 + var extractTimeMs by remember { mutableStateOf(0L) } 443 + var frameCount by remember { mutableStateOf(0) } 444 + 445 + // Controller to get feed dimensions from CameraView. 446 + val controller = remember { CameraViewControllerImpl() } 447 + var feedWidth by remember { mutableStateOf(0f) } 448 + var feedHeight by remember { mutableStateOf(0f) } 449 + // Actual CameraView display size in pixels (for cropOffsetY calculation) 450 + var cameraViewPxW by remember { mutableStateOf(0f) } 451 + var cameraViewPxH by remember { mutableStateOf(0f) } 452 + 453 + // Saved analysis results during recording, keyed by timestamp ms. 454 + val analysisResults = remember { mutableMapOf<Long, AnalysisResult>() } 455 + // Wall-clock time when recording starts. 456 + var videoStartWallClock by remember { mutableStateOf(0L) } 457 + var recordingActive by remember { mutableStateOf(false) } 458 + 459 + // Deferred to wait for video save callback 460 + val videoSavedDeferred = remember { mutableStateOf<CompletableDeferred<String>?>(null) } 461 + 462 + val coroutineScope = rememberCoroutineScope() 463 + 464 + PermissionProvider().apply { 465 + if (!hasCameraPermission()) RequestCameraPermission(onGranted = { 466 + permissionGranted = true 467 + }, onDenied = { permissionGranted = false }) else permissionGranted = true 468 + } 469 + 470 + // Save skeleton + objects together. Objects don't have their own timestamp, 471 + // so we cache the latest objects and attach them when a skeleton arrives. 472 + var latestObjects by remember { mutableStateOf<List<AnalysisObject>>(emptyList()) } 473 + LaunchedEffect(Unit) { 474 + customObjectRepository.customObjectFlow.collect { objs -> 475 + if (objs != null) latestObjects = objs 476 + } 477 + } 478 + LaunchedEffect(Unit) { 479 + skeletonRepository.skeletonFlow.collect { skel -> 480 + // Only save skeletons after recording has started (videoStartWallClock set by onRecordToggled) 481 + if (recordingActive && videoStartWallClock > 0 && skel != null) { 482 + analysisResults[skel.timestamp] = AnalysisResult(skel, latestObjects) 483 + } 484 + } 485 + } 486 + 487 + // Go to preview once permission is granted 488 + LaunchedEffect(permissionGranted) { 489 + if (permissionGranted && phase == "WAITING_PERMISSION") { 490 + phase = "PREVIEW" 491 + statusText = "Tap Record to start" 492 + } 493 + } 494 + 495 + // Recording trigger — launched when phase becomes RECORDING 496 + LaunchedEffect(phase) { 497 + if (phase == "RECORDING") { 498 + statusText = "Recording 3 seconds..." 499 + analysisResults.clear() 500 + val deferred = CompletableDeferred<String>() 501 + videoSavedDeferred.value = deferred 502 + val recordStart = Clock.System.now().toEpochMilliseconds() 503 + videoStartWallClock = 0L 504 + recordingActive = false 505 + val beforeSetId = System.currentTimeMillis() 506 + println("RECORD-TIMING: before setRecordingId wallMs=$beforeSetId") 507 + recordingId = "${Clock.System.now().epochSeconds}" 508 + val afterSetId = System.currentTimeMillis() 509 + println("RECORD-TIMING: after setRecordingId wallMs=$afterSetId delta=${afterSetId - beforeSetId}ms") 510 + 511 + delay(1500) 512 + 513 + recordingId = null 514 + statusText = "Waiting for video to save..." 515 + val savedPath = deferred.await() 516 + recordTimeMs = Clock.System.now().toEpochMilliseconds() - recordStart 517 + 518 + val goodSkeletons = analysisResults.values.count { it.skeleton != null && it.skeleton!!.joints().size >= 10 } 519 + println("DEBUG: skeleton data: $goodSkeletons good skeletons") 520 + 521 + originalVideoPath = savedPath 522 + phase = "COMPARING" 523 + statusText = "Analysing timestamp relationship..." 524 + 525 + val capturedAnalysisResults = analysisResults.toMap() 526 + val capturedVideoStart = videoStartWallClock 527 + 528 + val extractJob = coroutineScope.launch(kotlinx.coroutines.Dispatchers.Default) { 529 + val timestamps = listVideoFrameTimestamps(savedPath) 530 + val savedKeys = capturedAnalysisResults.keys.sorted() 531 + val firstTs = timestamps.firstOrNull() ?: return@launch 532 + println("DEBUG: ${timestamps.size} video frames, ${savedKeys.size} analysis results") 533 + println("DEBUG: videoStartWallClock=$capturedVideoStart") 534 + println("DEBUG: video PTS range: ${timestamps.first()}..${timestamps.last()} span=${timestamps.last() - timestamps.first()}ms") 535 + println("DEBUG: skeleton ts range: ${savedKeys.firstOrNull()}..${savedKeys.lastOrNull()} span=${if (savedKeys.size > 1) savedKeys.last() - savedKeys.first() else 0}ms") 536 + 537 + // Diagnostic: compare video PTS against skeleton timestamps 538 + // Video PTS is relative (starts from ~0). Skeleton timestamps are wall-clock. 539 + // If we subtract videoStartWallClock from skeleton timestamps, they should 540 + // become comparable to video PTS. The difference reveals the lag. 541 + val relativeKeys = savedKeys.map { it - capturedVideoStart } 542 + println("DIAG: relative skeleton timestamps (first 10): ${relativeKeys.take(10)}") 543 + println("DIAG: video PTS (first 10): ${timestamps.take(10)}") 544 + 545 + // For each skeleton, find the closest video PTS and report the difference 546 + val offsets = relativeKeys.mapNotNull { skelRelTs -> 547 + val closestPts = timestamps.minByOrNull { kotlin.math.abs(it - skelRelTs) } ?: return@mapNotNull null 548 + skelRelTs - closestPts 549 + } 550 + if (offsets.isNotEmpty()) { 551 + println("DIAG: skeleton-to-video offsets (first 10): ${offsets.take(10)}") 552 + println("DIAG: avg offset=${offsets.average().toLong()}ms min=${offsets.min()}ms max=${offsets.max()}ms") 553 + } 554 + 555 + // Pick a frame near 1.5s that has BOTH skeleton (10+ joints) AND objects 556 + val targetTime = capturedVideoStart + 750L 557 + val bestKey = savedKeys 558 + .filter { key -> 559 + val r = capturedAnalysisResults[key] 560 + r?.skeleton != null && r.skeleton!!.joints().size >= 10 && r.objects.isNotEmpty() 561 + } 562 + .minByOrNull { kotlin.math.abs(it - targetTime) } 563 + ?: savedKeys // fallback: just skeleton if no frame has both 564 + .filter { capturedAnalysisResults[it]?.skeleton?.joints()?.size ?: 0 >= 10 } 565 + .minByOrNull { kotlin.math.abs(it - targetTime) } 566 + val liveResult = bestKey?.let { capturedAnalysisResults[it] } 567 + val liveSkel = liveResult?.skeleton 568 + val liveObjects = liveResult?.objects ?: emptyList() 569 + if (liveSkel == null || liveSkel.joints().size < 10) { 570 + println("DEBUG: no good skeleton found") 571 + phase = "DONE" 572 + statusText = "No skeleton with objects found" 573 + return@launch 574 + } 575 + println("DEBUG: selected frame has ${liveSkel.joints().size} joints, ${liveObjects.size} objects") 576 + val skelVideoTs = bestKey - capturedVideoStart 577 + // Log all skeleton timestamps near the target for debugging 578 + val nearKeys = savedKeys.filter { kotlin.math.abs(it - targetTime) < 500 } 579 + println("DEBUG: skeleton at sensorTs=$bestKey videoTs=${skelVideoTs}ms joints=${liveSkel.joints().size}") 580 + println("DEBUG: videoStartWallClock=$capturedVideoStart targetTime=$targetTime") 581 + println("DEBUG: nearby skeleton keys (within 500ms of target): ${nearKeys.map { it - capturedVideoStart }}") 582 + println("DEBUG: first 5 skeleton keys (relative): ${savedKeys.take(5).map { it - capturedVideoStart }}") 583 + println("DEBUG: video PTS first=${timestamps.first()} last=${timestamps.last()} count=${timestamps.size}") 584 + // Show the gap between video start and first skeleton 585 + println("DEBUG: gap from recording start to first skeleton: ${savedKeys.first() - capturedVideoStart}ms") 586 + 587 + // Extract ALL frames and draw the same skeleton on each 588 + val results = mutableListOf<ComparisonResult>() 589 + var expectedPageIdx = 0 590 + 591 + extractFrames(savedPath, timestamps).collect { inputFrame -> 592 + val frameBitmap = inputFrame.toImageBitmap() 593 + val frameElapsed = inputFrame.timestamp - firstTs 594 + 595 + val output = ImageBitmap(frameBitmap.width, frameBitmap.height) 596 + val canvas = Canvas(output) 597 + val drawScope = CanvasDrawScope() 598 + drawScope.draw(Density(1f), LayoutDirection.Ltr, canvas, Size(frameBitmap.width.toFloat(), frameBitmap.height.toFloat())) { 599 + drawImage(frameBitmap) 600 + val fw = frameBitmap.width.toFloat(); val fh = frameBitmap.height.toFloat() 601 + val sA = liveSkel.width / liveSkel.height; val fA = fw / fh 602 + val coX: Float; val coY: Float; val sx: Float; val sy: Float 603 + if (kotlin.math.abs(sA - fA) > 0.01f) { 604 + if (sA > fA) { sy = fh / liveSkel.height; val vW = liveSkel.height * fA; coX = (liveSkel.width - vW) / 2f; sx = fw / vW; coY = 0f } 605 + else { sx = fw / liveSkel.width; val vH = liveSkel.width / fA; coY = (liveSkel.height - vH) / 2f; sy = fh / vH; coX = 0f } 606 + } else { sx = fw / liveSkel.width; sy = fh / liveSkel.height; coX = 0f; coY = 0f } 607 + fun pt(c: Skeleton.SkeletonCoordinate?) = c?.let { androidx.compose.ui.geometry.Offset((it.x - coX) * sx, (it.y - coY) * sy) } 608 + val conns = listOf(liveSkel.leftShoulder to liveSkel.leftElbow, liveSkel.leftElbow to liveSkel.leftWrist, liveSkel.rightShoulder to liveSkel.rightElbow, liveSkel.rightElbow to liveSkel.rightWrist, liveSkel.leftShoulder to liveSkel.rightShoulder, liveSkel.leftShoulder to liveSkel.leftHip, liveSkel.rightShoulder to liveSkel.rightHip, liveSkel.leftHip to liveSkel.rightHip, liveSkel.leftHip to liveSkel.leftKnee, liveSkel.leftKnee to liveSkel.leftAnkle, liveSkel.rightHip to liveSkel.rightKnee, liveSkel.rightKnee to liveSkel.rightAnkle) 609 + for ((f, t) in conns) { val p1 = pt(f); val p2 = pt(t); if (p1 != null && p2 != null) drawLine(Color.Blue, p1, p2, strokeWidth = 4f) } 610 + for (j in listOfNotNull(liveSkel.leftShoulder, liveSkel.rightShoulder, liveSkel.leftElbow, liveSkel.rightElbow, liveSkel.leftWrist, liveSkel.rightWrist, liveSkel.leftHip, liveSkel.rightHip, liveSkel.leftKnee, liveSkel.rightKnee, liveSkel.leftAnkle, liveSkel.rightAnkle)) { pt(j)?.let { drawCircle(Color.Blue, 6f, it) } } 611 + // Draw objects (green rectangles) with same aspect ratio correction 612 + for (obj in liveObjects) { 613 + val ofw = obj.frameSize.width.toFloat() 614 + val ofh = obj.frameSize.height.toFloat() 615 + val oA = ofw / ofh 616 + val ocX: Float; val ocY: Float; val osx: Float; val osy: Float 617 + if (kotlin.math.abs(oA - fA) > 0.01f) { 618 + if (oA > fA) { osy = fh / ofh; val vW = ofh * fA; ocX = (ofw - vW) / 2f; osx = fw / vW; ocY = 0f } 619 + else { osx = fw / ofw; val vH = ofw / fA; ocY = (ofh - vH) / 2f; osy = fh / vH; ocX = 0f } 620 + } else { osx = fw / ofw; osy = fh / ofh; ocX = 0f; ocY = 0f } 621 + val left = (obj.boundingBox.left - ocX) * osx 622 + val top = (obj.boundingBox.top - ocY) * osy 623 + val right = (obj.boundingBox.right - ocX) * osx 624 + val bottom = (obj.boundingBox.bottom - ocY) * osy 625 + drawRect(Color.Green, topLeft = androidx.compose.ui.geometry.Offset(left, top), size = Size(right - left, bottom - top), style = Stroke(3f)) 626 + } 627 + } 628 + 629 + if (frameElapsed <= skelVideoTs) expectedPageIdx = results.size 630 + val offset = frameElapsed - skelVideoTs 631 + results.add(ComparisonResult( 632 + image = output, 633 + label = "${frameElapsed}ms (${if (offset >= 0) "+" else ""}${offset}ms)", 634 + frameTs = inputFrame.timestamp, frameElapsedMs = frameElapsed, 635 + expectedElapsedMs = skelVideoTs, liveSkeleton = liveSkel, 636 + videoPath = savedPath, allTimestamps = timestamps, 637 + angleMatch = -1f, posMatch = -1f 638 + )) 639 + statusText = "Frame ${results.size}/${timestamps.size}..." 640 + } 641 + 642 + println("DEBUG: ${results.size} frames, expected page=$expectedPageIdx (${skelVideoTs}ms)") 643 + comparisonImages = results 644 + findBestResult = expectedPageIdx to emptyList() 645 + phase = "DONE" 646 + statusText = "Skeleton from ${skelVideoTs}ms — swipe to find match" 647 + } // end of launch(Dispatchers.Default) 648 + } 649 + } 650 + 651 + Column(modifier = Modifier.fillMaxSize()) { 652 + // Status bar 653 + Text( 654 + text = statusText, 655 + modifier = Modifier.fillMaxWidth().padding(8.dp), 656 + textAlign = TextAlign.Center, 657 + style = androidx.compose.material3.MaterialTheme.typography.titleSmall, 658 + ) 659 + 660 + // Timing info 661 + if (phase == "DONE") { 662 + val recordSec = (recordTimeMs / 100.0).roundToLong() / 10.0 663 + val extractSec = (extractTimeMs / 100.0).roundToLong() / 10.0 664 + val fps = if (extractTimeMs > 0) (frameCount * 1000.0 / extractTimeMs).roundToLong() else 0 665 + Text( 666 + text = "Record: ${recordSec}s | Extract+encode: ${extractSec}s ($frameCount frames, ~${fps} fps)", 667 + modifier = Modifier.fillMaxWidth().padding(horizontal = 8.dp), 668 + textAlign = TextAlign.Center, 669 + fontSize = 12.sp, 670 + ) 671 + } 672 + 673 + when (phase) { 674 + "WAITING_PERMISSION" -> { 675 + Box(modifier = Modifier.weight(1f).fillMaxWidth(), contentAlignment = Alignment.Center) { 676 + CircularProgressIndicator() 677 + } 678 + } 679 + "PREVIEW", "RECORDING" -> { 680 + if (permissionGranted) { 681 + var menuExpanded by remember { mutableStateOf(false) } 682 + DetectOrientation { orientation -> 683 + Box(modifier = Modifier.weight(1f).fillMaxWidth()) { 684 + key(orientation, detectMode) { 685 + CameraView( 686 + skeletonRepository = skeletonRepository, 687 + customObjectRepository = customObjectRepository, 688 + detectMode = detectMode, 689 + drawSkeleton = true, 690 + objectModel = generalModel, 691 + modifier = Modifier.fillMaxSize() 692 + .onSizeChanged { size -> 693 + cameraViewPxW = size.width.toFloat() 694 + cameraViewPxH = size.height.toFloat() 695 + }, 696 + frontCamera = false, 697 + recordingId = recordingId, 698 + controller = controller, 699 + onRecordToggled = { recording -> 700 + recordingActive = recording 701 + if (recording) { 702 + val callbackWallMs = System.currentTimeMillis() 703 + videoStartWallClock = callbackWallMs 704 + println("RECORD-TIMING: onRecordToggled(true) wallMs=$callbackWallMs") 705 + controller.requestData { data -> 706 + feedWidth = data.width 707 + feedHeight = data.height 708 + } 709 + } 710 + }, 711 + onVideoSaved = { id, url -> 712 + videoSavedDeferred.value?.complete(url) 713 + }, 714 + ) 715 + } 716 + // Menu overlay (same as CameraSample) 717 + if (phase == "PREVIEW") { 718 + Box(modifier = Modifier.padding(12.dp).align(Alignment.TopEnd)) { 719 + IconButton(onClick = { menuExpanded = true }) { 720 + Text("⋮", color = Color.White, fontSize = 22.sp) 721 + } 722 + DropdownMenu(expanded = menuExpanded, onDismissRequest = { menuExpanded = false }) { 723 + // Detection mode 724 + DropdownMenuItem(text = { 725 + Column { 726 + Text("Detection Mode", modifier = Modifier.fillMaxWidth(), textAlign = TextAlign.Center) 727 + listOf("Pose" to DetectMode.POSE, "Objects" to DetectMode.OBJECT, "Both" to DetectMode.BOTH, "None" to DetectMode.NONE).forEach { (label, mode) -> 728 + Row(horizontalArrangement = Arrangement.SpaceBetween, verticalAlignment = Alignment.CenterVertically, modifier = Modifier.fillMaxWidth()) { 729 + Text(label); Spacer(Modifier.width(12.dp)) 730 + RadioButton(selected = detectMode == mode, onClick = { detectMode = mode }) 731 + } 732 + } 733 + } 734 + }, onClick = {}) 735 + HorizontalDivider() 736 + // Model picker 737 + DropdownMenuItem(text = { 738 + Column { 739 + Text("Model", modifier = Modifier.fillMaxWidth(), textAlign = TextAlign.Center) 740 + availableModels.forEach { model -> 741 + Row(horizontalArrangement = Arrangement.SpaceBetween, verticalAlignment = Alignment.CenterVertically, modifier = Modifier.fillMaxWidth()) { 742 + Text(model.name, fontSize = 13.sp); Spacer(Modifier.width(12.dp)) 743 + RadioButton(selected = selectedModel == model, onClick = { selectedModel = model }) 744 + } 745 + } 746 + } 747 + }, onClick = {}) 748 + HorizontalDivider() 749 + // Record button 750 + DropdownMenuItem( 751 + text = { Text("Record 1.5 seconds") }, 752 + onClick = { menuExpanded = false; phase = "RECORDING" } 753 + ) 754 + } 755 + } 756 + } 757 + } 758 + } 759 + } 760 + } 761 + "EXTRACTING" -> { 762 + Box(modifier = Modifier.weight(1f).fillMaxWidth(), contentAlignment = Alignment.Center) { 763 + CircularProgressIndicator() 764 + } 765 + } 766 + "COMPARING" -> { 767 + Box(modifier = Modifier.weight(1f).fillMaxWidth(), contentAlignment = Alignment.Center) { 768 + CircularProgressIndicator() 769 + } 770 + } 771 + "DONE" -> { 772 + if (comparisonImages.isNotEmpty()) { 773 + val initialPage = findBestResult?.first ?: 0 774 + val pagerState = rememberPagerState(initialPage = initialPage, pageCount = { comparisonImages.size }) 775 + Column( 776 + modifier = Modifier.weight(1f).fillMaxWidth().padding(8.dp), 777 + verticalArrangement = Arrangement.spacedBy(4.dp) 778 + ) { 779 + val comp = comparisonImages.getOrNull(pagerState.currentPage) 780 + val lagText = "" 781 + Text( 782 + "${lagText}${pagerState.currentPage + 1}/${comparisonImages.size} — ${comp?.label ?: ""}", 783 + fontSize = 12.sp, textAlign = TextAlign.Center, modifier = Modifier.fillMaxWidth() 784 + ) 785 + HorizontalPager( 786 + state = pagerState, 787 + modifier = Modifier.weight(1f).fillMaxWidth() 788 + ) { page -> 789 + val c = comparisonImages[page] 790 + Image( 791 + bitmap = c.image, 792 + contentDescription = c.label, 793 + contentScale = ContentScale.Fit, 794 + modifier = Modifier.fillMaxSize() 795 + ) 796 + } 797 + Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(8.dp)) { 798 + androidx.compose.material3.Button( 799 + onClick = { 800 + val c = comparisonImages.getOrNull(pagerState.currentPage) 801 + if (c != null) { 802 + val lagMs = c.frameElapsedMs - c.expectedElapsedMs 803 + println("BEST-MATCH: lagMs=$lagMs frameElapsed=${c.frameElapsedMs}ms expected=${c.expectedElapsedMs}ms mode=$detectMode model=${selectedModel?.name} page=${pagerState.currentPage}/${comparisonImages.size}") 804 + } 805 + }, 806 + modifier = Modifier.weight(1f) 807 + ) { Text("Best Match") } 808 + androidx.compose.material3.Button( 809 + onClick = { 810 + comparisonImages = emptyList() 811 + findBestResult = null 812 + phase = "PREVIEW" 813 + statusText = "Tap Record to start" 814 + }, 815 + modifier = Modifier.weight(1f) 816 + ) { Text("New Recording") } 817 + } 818 + } 819 + } else { 820 + Box(modifier = Modifier.weight(1f).fillMaxWidth(), contentAlignment = Alignment.Center) { 821 + Text("No comparison images produced") 822 + } 823 + } 824 + } 379 825 } 380 826 } 381 827 }

sample/iosApp/FastViTT8F16.mlpackage/Data/com.apple.CoreML/model.mlmodel

This is a binary file and will not be displayed.

sample/iosApp/FastViTT8F16.mlpackage/Data/com.apple.CoreML/weights/weight.bin

This is a binary file and will not be displayed.

-18

sample/iosApp/FastViTT8F16.mlpackage/Manifest.json

··· 1 - { 2 - "fileFormatVersion": "1.0.0", 3 - "itemInfoEntries": { 4 - "76187EC5-87E5-4263-B6E2-1CF5E747A0EE": { 5 - "author": "com.apple.CoreML", 6 - "description": "CoreML Model Weights", 7 - "name": "weights", 8 - "path": "com.apple.CoreML/weights" 9 - }, 10 - "D3756FF7-6CCB-4582-AB58-B91896E60AE4": { 11 - "author": "com.apple.CoreML", 12 - "description": "CoreML Model Specification", 13 - "name": "model.mlmodel", 14 - "path": "com.apple.CoreML/model.mlmodel" 15 - } 16 - }, 17 - "rootModelIdentifier": "D3756FF7-6CCB-4582-AB58-B91896E60AE4" 18 - }

+2 -6

sample/iosApp/iosApp.xcodeproj/project.pbxproj

··· 7 7 objects = { 8 8 9 9 /* Begin PBXBuildFile section */ 10 - 438D2B632DF4C5AC00625680 /* FastViTT8F16.mlpackage in Sources */ = {isa = PBXBuildFile; fileRef = 438D2B622DF4C5AC00625680 /* FastViTT8F16.mlpackage */; }; 11 10 A93A953B29CC810C00F8E227 /* iosApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = A93A953A29CC810C00F8E227 /* iosApp.swift */; }; 12 11 A93A953F29CC810D00F8E227 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = A93A953E29CC810D00F8E227 /* Assets.xcassets */; }; 13 12 A93A954229CC810D00F8E227 /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = A93A954129CC810D00F8E227 /* Preview Assets.xcassets */; }; 14 13 /* End PBXBuildFile section */ 15 14 16 15 /* Begin PBXFileReference section */ 17 - 438D2B622DF4C5AC00625680 /* FastViTT8F16.mlpackage */ = {isa = PBXFileReference; lastKnownFileType = folder.mlpackage; path = FastViTT8F16.mlpackage; sourceTree = "<group>"; }; 18 16 A93A953729CC810C00F8E227 /* PoseDetection.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = PoseDetection.app; sourceTree = BUILT_PRODUCTS_DIR; }; 19 17 A93A953A29CC810C00F8E227 /* iosApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = iosApp.swift; sourceTree = "<group>"; }; 20 18 A93A953E29CC810D00F8E227 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; }; ··· 75 73 C4127409AE3703430489E7BC /* Frameworks */ = { 76 74 isa = PBXGroup; 77 75 children = ( 78 - 438D2B622DF4C5AC00625680 /* FastViTT8F16.mlpackage */, 79 76 ); 80 77 name = Frameworks; 81 78 sourceTree = "<group>"; ··· 175 172 buildActionMask = 2147483647; 176 173 files = ( 177 174 A93A953B29CC810C00F8E227 /* iosApp.swift in Sources */, 178 - 438D2B632DF4C5AC00625680 /* FastViTT8F16.mlpackage in Sources */, 179 175 ); 180 176 runOnlyForDeploymentPostprocessing = 0; 181 177 }; ··· 306 302 CODE_SIGN_STYLE = Automatic; 307 303 CURRENT_PROJECT_VERSION = 1; 308 304 DEVELOPMENT_ASSET_PATHS = "\"iosApp/Preview Content\""; 309 - DEVELOPMENT_TEAM = FAGG2XS28P; 305 + DEVELOPMENT_TEAM = 6H9FHG23L3; 310 306 ENABLE_PREVIEWS = YES; 311 307 GENERATE_INFOPLIST_FILE = YES; 312 308 INFOPLIST_FILE = iosApp/Info.plist; ··· 332 328 CODE_SIGN_STYLE = Automatic; 333 329 CURRENT_PROJECT_VERSION = 1; 334 330 DEVELOPMENT_ASSET_PATHS = "\"iosApp/Preview Content\""; 335 - DEVELOPMENT_TEAM = FAGG2XS28P; 331 + DEVELOPMENT_TEAM = 6H9FHG23L3; 336 332 ENABLE_PREVIEWS = YES; 337 333 GENERATE_INFOPLIST_FILE = YES; 338 334 INFOPLIST_FILE = iosApp/Info.plist;

sample/iosApp/iosApp/models/YOLOv3FP16.mlmodel

This is a binary file and will not be displayed.

Configure Feed

Configure Feed