release: 4.12.1 — iOS rect-aspect detection (multiarray decode) · nateholland.bsky.social/PoseDetection@c08f22d

+1 -1

posedetection/build.gradle.kts

··· 4 4 5 5 mavenPublishing { 6 6 publishToMavenCentral(SonatypeHost.CENTRAL_PORTAL) 7 - coordinates("com.performancecoachlab.posedetection", "posedetection-compose", "4.11.0") 7 + coordinates("com.performancecoachlab.posedetection", "posedetection-compose", "4.12.1") 8 8 9 9 pom { 10 10 name.set("Pose Detection")

+114 -23

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt

··· 44 44 import platform.ImageIO.kCGImagePropertyOrientationRightMirrored 45 45 import platform.ImageIO.kCGImagePropertyOrientationUp 46 46 import platform.ImageIO.kCGImagePropertyOrientationUpMirrored 47 + import platform.CoreML.MLMultiArray 48 + import platform.Foundation.NSNumber 47 49 import platform.Vision.VNClassificationObservation 50 + import platform.Vision.VNCoreMLFeatureValueObservation 48 51 import platform.Vision.VNCoreMLModel 49 52 import platform.Vision.VNCoreMLRequest 53 + import kotlinx.cinterop.CPointer 54 + import kotlinx.cinterop.FloatVar 55 + import kotlinx.cinterop.get 56 + import kotlinx.cinterop.reinterpret 50 57 import platform.Vision.VNDetectHumanBodyPoseRequest 51 58 import platform.Vision.VNHumanBodyPoseObservation 52 59 import platform.Vision.VNHumanBodyPoseObservationJointName ··· 247 254 private var focusArea: Rect? = null 248 255 private var path = "" 249 256 private var detectMode = DetectMode.BOTH 257 + // Populated by setObjectModel from ObjectModel.inputWidth/Height (filename 258 + // suffix convention). Used by the raw multiarray decoder to map ultralytics 259 + // pixel-space output back into normalized source coords. 0 when unknown. 260 + private var modelInputW: Int = 0 261 + private var modelInputH: Int = 0 250 262 251 263 fun setFocusArea(focusArea: Rect?) { 252 264 this.focusArea = focusArea ··· 258 270 259 271 fun setObjectModel(objectModel: ObjectModel?) { 260 272 modelObj = objectModel?.getModel() 273 + modelInputW = objectModel?.inputWidth ?: 0 274 + modelInputH = objectModel?.inputHeight ?: 0 261 275 setUpRecognition() 262 276 } 263 277 ··· 511 525 val recognized = 512 526 results.filterIsInstance<VNRecognizedObjectObservation>() 513 527 514 - val analysisObjects = recognized.map { observation -> 515 - val boundingBox = observation.boundingBox.toOrientedPixelRect( 516 - rawWidth = rawWidth, 517 - rawHeight = rawHeight, 518 - exifOrientation = exifOrientation 519 - ) 520 - 521 - val labels = observation.labels.mapNotNull { 522 - (it as VNClassificationObservation).let { ca -> 523 - if (ca.confidence > 0.0) Label( 524 - ca.identifier, 525 - ca.confidence 526 - ) else null 528 + val analysisObjects = if (recognized.isNotEmpty()) { 529 + recognized.map { observation -> 530 + val boundingBox = 531 + observation.boundingBox.toOrientedPixelRect( 532 + rawWidth = rawWidth, 533 + rawHeight = rawHeight, 534 + exifOrientation = exifOrientation 535 + ) 536 + val labels = observation.labels.mapNotNull { 537 + (it as VNClassificationObservation).let { ca -> 538 + if (ca.confidence > 0.0) Label( 539 + ca.identifier, 540 + ca.confidence 541 + ) else null 542 + } 527 543 } 544 + AnalysisObject( 545 + trackingId = stableTrackingId(observation), 546 + labels = labels, 547 + boundingBox = boundingBox, 548 + frameSize = FrameSize( 549 + width = orientedSize.width.toInt().absoluteValue, 550 + height = orientedSize.height.toInt().absoluteValue 551 + ), 552 + timestamp = timestamp 553 + ) 528 554 } 529 - 530 - AnalysisObject( 531 - trackingId = stableTrackingId(observation), 532 - labels = labels, 533 - boundingBox = boundingBox, 534 - frameSize = FrameSize( 535 - width = orientedSize.width.toInt().absoluteValue, 536 - height = orientedSize.height.toInt().absoluteValue 537 - ), 555 + } else { 556 + // Raw multiarray output (e.g. yolo26n end2end). 557 + decodeRawMultiArrayDetections( 558 + results = results, 559 + modelInputW = modelInputW, 560 + modelInputH = modelInputH, 561 + orientedW = orientedSize.width, 562 + orientedH = orientedSize.height, 538 563 timestamp = timestamp 539 564 ) 540 565 } ··· 542 567 onObjectsProcessed(analysisObjects) 543 568 }.apply { 544 569 imageCropAndScaleOption = 545 - platform.Vision.VNImageCropAndScaleOptionCenterCrop 570 + platform.Vision.VNImageCropAndScaleOptionScaleFit 546 571 } 547 572 } 548 573 } else null ··· 812 837 813 838 private fun stableTrackingId(observation: VNRecognizedObjectObservation): Int = 814 839 observation.hashCode() 840 + 841 + private val RAW_CLASS_NAMES = listOf("basketball", "basketball_hoop") 842 + 843 + @OptIn(ExperimentalForeignApi::class) 844 + private fun decodeRawMultiArrayDetections( 845 + results: List<*>, 846 + modelInputW: Int, 847 + modelInputH: Int, 848 + orientedW: Float, 849 + orientedH: Float, 850 + timestamp: Long, 851 + ): List<AnalysisObject> { 852 + if (modelInputW <= 0 || modelInputH <= 0) return emptyList() 853 + val featureObs = results.filterIsInstance<VNCoreMLFeatureValueObservation>() 854 + val out = mutableListOf<AnalysisObject>() 855 + for (obs in featureObs) { 856 + val arr: MLMultiArray = obs.featureValue.multiArrayValue ?: continue 857 + val shape = arr.shape 858 + if (shape.size != 3) continue 859 + val dim1 = (shape[1] as NSNumber).intValue 860 + val dim2 = (shape[2] as NSNumber).intValue 861 + if (dim2 != 6) continue 862 + val dataPtr = arr.dataPointer?.reinterpret<FloatVar>() ?: continue 863 + 864 + // Use strides (element counts) to compute offsets — safe against 865 + // non-contiguous storage. 866 + val strides = arr.strides 867 + val rowStride = if (strides.size == 3) (strides[1] as NSNumber).intValue else 6 868 + val colStride = if (strides.size == 3) (strides[2] as NSNumber).intValue else 1 869 + fun at(i: Int, j: Int): Float = dataPtr[i * rowStride + j * colStride] 870 + 871 + for (i in 0 until dim1) { 872 + val conf = at(i, 4) 873 + if (conf <= 0.25f) continue 874 + val x1n = at(i, 0) 875 + val y1n = at(i, 1) 876 + val x2n = at(i, 2) 877 + val y2n = at(i, 3) 878 + val cls = at(i, 5).toInt() 879 + // Ultralytics end2end CoreML export emits pixel-space coordinates 880 + // over the model input. Normalize by dividing by the model input 881 + // dimensions (passed in from ObjectModel.inputWidth/Height) before 882 + // mapping to oriented source pixel space. 883 + val mW = modelInputW.toFloat() 884 + val mH = modelInputH.toFloat() 885 + val leftPx = (min(x1n, x2n) / mW) * orientedW 886 + val rightPx = (max(x1n, x2n) / mW) * orientedW 887 + val topPx = (min(y1n, y2n) / mH) * orientedH 888 + val bottomPx = (max(y1n, y2n) / mH) * orientedH 889 + val label = RAW_CLASS_NAMES.getOrNull(cls) ?: "class_$cls" 890 + out.add( 891 + AnalysisObject( 892 + trackingId = 0, 893 + labels = listOf(Label(label, conf)), 894 + boundingBox = Rect(leftPx, topPx, rightPx, bottomPx), 895 + frameSize = FrameSize( 896 + width = orientedW.toInt().absoluteValue, 897 + height = orientedH.toInt().absoluteValue 898 + ), 899 + timestamp = timestamp 900 + ) 901 + ) 902 + } 903 + } 904 + return out 905 + } 815 906 816 907 data class DetectedObject @OptIn(ExperimentalForeignApi::class) constructor( 817 908 val id: String = NSUUID().UUIDString(),

+25 -19

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/custom/CustomObjectModel.ios.kt

··· 9 9 10 10 @Composable 11 11 actual fun initialiseObjectModel(modelPath: ModelPath): ObjectModel { 12 - val model = createObjectDetector(modelPath.iosModelPath) 13 - return ObjectModel(model) 12 + return createObjectDetector(modelPath.iosModelPath) 14 13 } 15 14 16 15 @OptIn(ExperimentalForeignApi::class) 17 - fun createObjectDetector(model: String?): VNCoreMLModel? { 16 + fun createObjectDetector(model: String?): ObjectModel { 18 17 if (model == null) { 19 - return null 18 + return ObjectModel(null, 0, 0) 20 19 } 21 - //println("Model input: $model") 22 20 val path = NSBundle.mainBundle.pathForResource(model, "mlmodelc") 23 - //println("Model path: $path") 24 21 val url = path?.let { NSURL.fileURLWithPath(it) } 25 - //println("Model URL: $url") 26 - val modelCont = url?.let { MLModel.modelWithContentsOfURL(it, null) } 27 - //println("Model content: $modelCont") 28 - val modelObj = modelCont?.let { VNCoreMLModel.modelForMLModel(it, null) } 29 - //println("Model: $modelObj") 30 - return modelObj 22 + val mlModel = url?.let { MLModel.modelWithContentsOfURL(it, null) } 23 + val vnModel = mlModel?.let { VNCoreMLModel.modelForMLModel(it, null) } 24 + val (w, h) = inferInputSizeFromName(model) 25 + return ObjectModel(vnModel, w, h) 31 26 } 32 27 33 - actual class ObjectModel { 34 - private var model: VNCoreMLModel? = null 28 + // Infer the model's letterbox input size from the filename suffix. 29 + // Matches the `_<W>x<H>` convention used by the rect exports (e.g. 30 + // `yolo26n_v11_rect_512x384`). Models without this suffix get (0, 0), 31 + // which disables letterboxing and preserves prior Vision-default behavior. 32 + private val INPUT_SIZE_SUFFIX = Regex("_(\\d+)x(\\d+)$") 35 33 36 - constructor(model: VNCoreMLModel?) { 37 - this.model = model 34 + internal fun inferInputSizeFromName(name: String): Pair<Int, Int> { 35 + INPUT_SIZE_SUFFIX.find(name)?.let { m -> 36 + val w = m.groupValues[1].toIntOrNull() ?: return 0 to 0 37 + val h = m.groupValues[2].toIntOrNull() ?: return 0 to 0 38 + return w to h 38 39 } 40 + return 0 to 0 41 + } 39 42 40 - fun getModel(): VNCoreMLModel? { 41 - return model 42 - } 43 + actual class ObjectModel( 44 + private val model: VNCoreMLModel?, 45 + val inputWidth: Int, 46 + val inputHeight: Int, 47 + ) { 48 + fun getModel(): VNCoreMLModel? = model 43 49 } 44 50 45 51 @Composable

+220 -55

posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/custom/ImageDetector.ios.kt

··· 6 6 import com.performancecoachlab.posedetection.recording.AnalysisObject 7 7 import com.performancecoachlab.posedetection.recording.FrameSize 8 8 import com.performancecoachlab.posedetection.recording.Label 9 + import kotlinx.cinterop.CPointer 9 10 import kotlinx.cinterop.ExperimentalForeignApi 11 + import kotlinx.cinterop.FloatVar 10 12 import kotlinx.cinterop.addressOf 13 + import kotlinx.cinterop.get 14 + import kotlinx.cinterop.reinterpret 11 15 import kotlinx.cinterop.usePinned 12 16 import platform.CoreGraphics.* 17 + import platform.CoreML.MLMultiArray 18 + import platform.Foundation.NSNumber 19 + import platform.Vision.VNCoreMLFeatureValueObservation 13 20 import platform.Vision.VNCoreMLRequest 21 + import platform.Vision.VNImageCropAndScaleOptionScaleFit 14 22 import platform.Vision.VNImageRequestHandler 15 23 import platform.Vision.VNRecognizedObjectObservation 24 + import kotlin.math.max 25 + import kotlin.math.min 16 26 17 27 @OptIn(ExperimentalForeignApi::class) 18 28 actual class ImageDetector actual constructor(model: ObjectModel) { 19 29 20 30 private val vncoreModel = model.getModel() 31 + private val inputW = model.inputWidth 32 + private val inputH = model.inputHeight 21 33 22 34 actual fun detect(image: ImageBitmap): List<AnalysisObject> { 23 35 val model = vncoreModel ?: return emptyList() 24 - val w = image.width 25 - val h = image.height 36 + val imgW = image.width 37 + val imgH = image.height 38 + if (imgW <= 0 || imgH <= 0) return emptyList() 26 39 27 - val pixelMap = image.toPixelMap() 28 - val buffer = ByteArray(w * h * 4) 29 - for (y in 0 until h) { 30 - for (x in 0 until w) { 31 - val color = pixelMap[x, y] 32 - val idx = (y * w + x) * 4 33 - buffer[idx] = (color.red * 255).toInt().toByte() 34 - buffer[idx + 1] = (color.green * 255).toInt().toByte() 35 - buffer[idx + 2] = (color.blue * 255).toInt().toByte() 36 - buffer[idx + 3] = (color.alpha * 255).toInt().toByte() 37 - } 38 - } 40 + val srcCgImage = createCgImageFromImageBitmap(image) ?: return emptyList() 39 41 40 - val colorSpace = CGColorSpaceCreateDeviceRGB() 41 - val bitmapInfo = CGImageAlphaInfo.kCGImageAlphaPremultipliedLast.value 42 - val context = CGBitmapContextCreate( 43 - null, w.toULong(), h.toULong(), 8u, 44 - (w * 4).toULong(), colorSpace, bitmapInfo 45 - ) ?: return emptyList() 46 - 47 - buffer.usePinned { pinned -> 48 - val data = CGBitmapContextGetData(context) 49 - if (data != null) { 50 - platform.posix.memcpy(data, pinned.addressOf(0), (w * h * 4).toULong()) 51 - } 52 - } 53 - 54 - val cgImage = CGBitmapContextCreateImage(context) ?: run { 55 - CGContextRelease(context) 56 - CGColorSpaceRelease(colorSpace) 57 - return emptyList() 42 + // If we know the model's input size, letterbox the source into 43 + // (inputW × inputH) with gray-114 padding — matching ultralytics' 44 + // training-time preprocessing. Otherwise fall back to passing the 45 + // raw image and letting Vision scale it. 46 + val useLetterbox = inputW > 0 && inputH > 0 47 + val scale: Float 48 + val padX: Float 49 + val padY: Float 50 + val handlerImage: CPointer<cnames.structs.CGImage> 51 + if (useLetterbox) { 52 + val s = min( 53 + inputW.toFloat() / imgW.toFloat(), 54 + inputH.toFloat() / imgH.toFloat() 55 + ) 56 + val scaledW = (imgW * s) 57 + val scaledH = (imgH * s) 58 + val px = (inputW - scaledW) / 2f 59 + val py = (inputH - scaledH) / 2f 60 + val letterboxed = createLetterboxedCgImage(srcCgImage, inputW, inputH, px, py, scaledW, scaledH) 61 + CGImageRelease(srcCgImage) 62 + if (letterboxed == null) return emptyList() 63 + scale = s 64 + padX = px 65 + padY = py 66 + handlerImage = letterboxed 67 + } else { 68 + scale = 1f 69 + padX = 0f 70 + padY = 0f 71 + handlerImage = srcCgImage 58 72 } 59 73 60 74 val results = mutableListOf<AnalysisObject>() 61 - val request = VNCoreMLRequest(model) { req, error -> 75 + val bbW = if (useLetterbox) inputW.toFloat() else imgW.toFloat() 76 + val bbH = if (useLetterbox) inputH.toFloat() else imgH.toFloat() 77 + val request = VNCoreMLRequest(model) { req, _ -> 62 78 val observations = req?.results as? List<*> ?: return@VNCoreMLRequest 63 - for (obs in observations.filterIsInstance<VNRecognizedObjectObservation>()) { 64 - val label = obs.labels.firstOrNull()?.toString() ?: "Unknown" 65 - val confidence = obs.confidence 66 - val bb = obs.boundingBox 67 - // Vision coordinates: origin bottom-left, normalized 0-1 68 - val left = CGRectGetMinX(bb).toFloat() * w 69 - val top = (1f - CGRectGetMaxY(bb).toFloat()) * h 70 - val right = CGRectGetMaxX(bb).toFloat() * w 71 - val bottom = (1f - CGRectGetMinY(bb).toFloat()) * h 72 - results.add( 73 - AnalysisObject( 74 - boundingBox = Rect(left, top, right, bottom), 75 - trackingId = 0, 76 - labels = listOf(Label(label, confidence)), 77 - frameSize = FrameSize(w, h), 78 - timestamp = 0L 79 - ) 80 - ) 79 + for (obs in observations) { 80 + when (obs) { 81 + is VNRecognizedObjectObservation -> { 82 + // Vision-compatible model (classifier + coordinates pipeline). 83 + // Coords: origin bottom-left, normalized 0-1 over the handler image. 84 + val label = obs.labels.firstOrNull()?.toString() ?: "Unknown" 85 + val confidence = obs.confidence 86 + val bb = obs.boundingBox 87 + val x1LB = CGRectGetMinX(bb).toFloat() * bbW 88 + val y1LB = (1f - CGRectGetMaxY(bb).toFloat()) * bbH 89 + val x2LB = CGRectGetMaxX(bb).toFloat() * bbW 90 + val y2LB = (1f - CGRectGetMinY(bb).toFloat()) * bbH 91 + addDetection( 92 + results, x1LB, y1LB, x2LB, y2LB, padX, padY, scale, 93 + imgW, imgH, label, confidence 94 + ) 95 + } 96 + is VNCoreMLFeatureValueObservation -> { 97 + // Raw multiarray output (e.g. yolo26n end2end). Shape 98 + // [1, 300, 6] with rows [x1, y1, x2, y2, conf, cls] in 99 + // normalized top-left-origin coords over the handler image. 100 + val arr = obs.featureValue.multiArrayValue ?: continue 101 + decodeRawDetections( 102 + arr, results, bbW, bbH, padX, padY, scale, imgW, imgH 103 + ) 104 + } 105 + } 81 106 } 107 + }.apply { 108 + // Belt-and-suspenders: input already matches model aspect when 109 + // letterboxed, so this is effectively a no-op — but prevents 110 + // Vision from re-cropping if anything upstream shifts. 111 + imageCropAndScaleOption = VNImageCropAndScaleOptionScaleFit 82 112 } 83 113 84 - val handler = VNImageRequestHandler(cgImage, mapOf<Any?, Any?>()) 114 + val handler = VNImageRequestHandler(handlerImage, mapOf<Any?, Any?>()) 85 115 handler.performRequests(listOf(request), null) 86 116 87 - CGImageRelease(cgImage) 88 - CGContextRelease(context) 117 + CGImageRelease(handlerImage) 118 + return results 119 + } 120 + } 121 + 122 + private fun addDetection( 123 + results: MutableList<AnalysisObject>, 124 + x1LB: Float, y1LB: Float, x2LB: Float, y2LB: Float, 125 + padX: Float, padY: Float, scale: Float, 126 + imgW: Int, imgH: Int, 127 + label: String, confidence: Float, 128 + ) { 129 + val left = ((x1LB - padX) / scale).coerceIn(0f, imgW.toFloat()) 130 + val top = ((y1LB - padY) / scale).coerceIn(0f, imgH.toFloat()) 131 + val right = ((x2LB - padX) / scale).coerceIn(0f, imgW.toFloat()) 132 + val bottom = ((y2LB - padY) / scale).coerceIn(0f, imgH.toFloat()) 133 + results.add( 134 + AnalysisObject( 135 + boundingBox = Rect(left, top, right, bottom), 136 + trackingId = 0, 137 + labels = listOf(Label(label, confidence)), 138 + frameSize = FrameSize(imgW, imgH), 139 + timestamp = 0L 140 + ) 141 + ) 142 + } 143 + 144 + private val CLASS_NAMES = listOf("basketball", "basketball_hoop") 145 + 146 + @OptIn(ExperimentalForeignApi::class) 147 + private fun decodeRawDetections( 148 + arr: MLMultiArray, 149 + results: MutableList<AnalysisObject>, 150 + bbW: Float, bbH: Float, 151 + padX: Float, padY: Float, scale: Float, 152 + imgW: Int, imgH: Int, 153 + ) { 154 + val shape = arr.shape 155 + if (shape.size != 3) return 156 + val dim1 = (shape[1] as NSNumber).intValue 157 + val dim2 = (shape[2] as NSNumber).intValue 158 + // Expect [1, N, 6] (yolo end2end). 159 + if (dim2 != 6) return 160 + val elements = dim1 161 + val dataPtr = arr.dataPointer?.reinterpret<FloatVar>() ?: return 162 + fun at(i: Int, j: Int): Float = dataPtr[i * 6 + j] 163 + for (i in 0 until elements) { 164 + val conf = at(i, 4) 165 + if (conf <= 0.25f) continue 166 + val x1n = at(i, 0) 167 + val y1n = at(i, 1) 168 + val x2n = at(i, 2) 169 + val y2n = at(i, 3) 170 + val cls = at(i, 5).toInt() 171 + // Top-left-origin normalized coords over the handler image. 172 + val x1LB = min(x1n, x2n) * bbW 173 + val y1LB = min(y1n, y2n) * bbH 174 + val x2LB = max(x1n, x2n) * bbW 175 + val y2LB = max(y1n, y2n) * bbH 176 + val label = CLASS_NAMES.getOrNull(cls) ?: "class_$cls" 177 + addDetection( 178 + results, x1LB, y1LB, x2LB, y2LB, padX, padY, scale, 179 + imgW, imgH, label, conf 180 + ) 181 + } 182 + } 183 + 184 + @OptIn(ExperimentalForeignApi::class) 185 + private fun createCgImageFromImageBitmap(image: ImageBitmap): CPointer<cnames.structs.CGImage>? { 186 + val w = image.width 187 + val h = image.height 188 + val pixelMap = image.toPixelMap() 189 + val buffer = ByteArray(w * h * 4) 190 + for (y in 0 until h) { 191 + for (x in 0 until w) { 192 + val color = pixelMap[x, y] 193 + val idx = (y * w + x) * 4 194 + buffer[idx] = (color.red * 255).toInt().toByte() 195 + buffer[idx + 1] = (color.green * 255).toInt().toByte() 196 + buffer[idx + 2] = (color.blue * 255).toInt().toByte() 197 + buffer[idx + 3] = (color.alpha * 255).toInt().toByte() 198 + } 199 + } 200 + val colorSpace = CGColorSpaceCreateDeviceRGB() 201 + val bitmapInfo = CGImageAlphaInfo.kCGImageAlphaPremultipliedLast.value 202 + val context = CGBitmapContextCreate( 203 + null, w.toULong(), h.toULong(), 8u, 204 + (w * 4).toULong(), colorSpace, bitmapInfo 205 + ) ?: run { 89 206 CGColorSpaceRelease(colorSpace) 207 + return null 208 + } 209 + buffer.usePinned { pinned -> 210 + val data = CGBitmapContextGetData(context) 211 + if (data != null) { 212 + platform.posix.memcpy(data, pinned.addressOf(0), (w * h * 4).toULong()) 213 + } 214 + } 215 + val cgImage = CGBitmapContextCreateImage(context) 216 + CGContextRelease(context) 217 + CGColorSpaceRelease(colorSpace) 218 + return cgImage 219 + } 90 220 91 - return results 221 + @OptIn(ExperimentalForeignApi::class) 222 + private fun createLetterboxedCgImage( 223 + src: CPointer<cnames.structs.CGImage>, 224 + outW: Int, 225 + outH: Int, 226 + padX: Float, 227 + padY: Float, 228 + drawW: Float, 229 + drawH: Float, 230 + ): CPointer<cnames.structs.CGImage>? { 231 + val colorSpace = CGColorSpaceCreateDeviceRGB() 232 + val bitmapInfo = CGImageAlphaInfo.kCGImageAlphaPremultipliedLast.value 233 + val ctx = CGBitmapContextCreate( 234 + null, outW.toULong(), outH.toULong(), 8u, 235 + (outW * 4).toULong(), colorSpace, bitmapInfo 236 + ) ?: run { 237 + CGColorSpaceRelease(colorSpace) 238 + return null 92 239 } 240 + 241 + // Gray-114 fill to match ultralytics letterbox padding. 242 + CGContextSetRGBFillColor(ctx, 114.0 / 255.0, 114.0 / 255.0, 114.0 / 255.0, 1.0) 243 + CGContextFillRect(ctx, CGRectMake(0.0, 0.0, outW.toDouble(), outH.toDouble())) 244 + 245 + // Core Graphics has bottom-left origin; our padY/drawH are top-left, 246 + // so flip the Y coordinate before drawing. 247 + val flippedY = outH.toFloat() - padY - drawH 248 + CGContextDrawImage( 249 + ctx, 250 + CGRectMake(padX.toDouble(), flippedY.toDouble(), drawW.toDouble(), drawH.toDouble()), 251 + src 252 + ) 253 + 254 + val image = CGBitmapContextCreateImage(ctx) 255 + CGContextRelease(ctx) 256 + CGColorSpaceRelease(colorSpace) 257 + return image 93 258 }

+3 -3

sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt

··· 505 505 val recordStart = Clock.System.now().toEpochMilliseconds() 506 506 videoStartWallClock = 0L 507 507 recordingActive = false 508 - val beforeSetId = System.currentTimeMillis() 508 + val beforeSetId = Clock.System.now().toEpochMilliseconds() 509 509 println("RECORD-TIMING: before setRecordingId wallMs=$beforeSetId") 510 510 recordingId = "${Clock.System.now().epochSeconds}" 511 - val afterSetId = System.currentTimeMillis() 511 + val afterSetId = Clock.System.now().toEpochMilliseconds() 512 512 println("RECORD-TIMING: after setRecordingId wallMs=$afterSetId delta=${afterSetId - beforeSetId}ms") 513 513 514 514 delay(1500) ··· 702 702 onRecordToggled = { recording -> 703 703 recordingActive = recording 704 704 if (recording) { 705 - val callbackWallMs = System.currentTimeMillis() 705 + val callbackWallMs = Clock.System.now().toEpochMilliseconds() 706 706 videoStartWallClock = callbackWallMs 707 707 println("RECORD-TIMING: onRecordToggled(true) wallMs=$callbackWallMs") 708 708 controller.requestData { data ->

+10 -2

sample/composeApp/src/iosMain/kotlin/com/nate/posedetection/ExperimentAuto.ios.kt

··· 2 2 3 3 import androidx.compose.runtime.Composable 4 4 import androidx.compose.runtime.remember 5 + import platform.Foundation.NSLog 6 + import platform.posix.exit 5 7 6 8 private object IosExperimentAutoBridge : ExperimentAutoBridge { 7 - override fun log(msg: String) {} 8 - override fun finishActivity() {} 9 + override fun log(msg: String) { 10 + NSLog("[ExperimentAuto] %s", msg) 11 + } 12 + 13 + override fun finishActivity() { 14 + NSLog("[ExperimentAuto] finishActivity — exiting") 15 + exit(0) 16 + } 9 17 } 10 18 11 19 @Composable

+31 -4

sample/composeApp/src/iosMain/kotlin/com/nate/posedetection/ExperimentLogger.ios.kt

··· 2 2 3 3 import androidx.compose.runtime.Composable 4 4 import androidx.compose.runtime.remember 5 + import platform.Foundation.NSDocumentDirectory 6 + import platform.Foundation.NSFileManager 7 + import platform.Foundation.NSSearchPathForDirectoriesInDomains 8 + import platform.Foundation.NSString 9 + import platform.Foundation.NSUserDomainMask 10 + import platform.Foundation.NSUTF8StringEncoding 11 + import platform.Foundation.writeToFile 12 + import platform.UIKit.UIDevice 5 13 6 - private object IosExperimentLogger : ExperimentLogger { 7 - override val deviceLabel: String = "ios" 8 - override fun write(filename: String, json: String): String? = null 14 + @OptIn(kotlinx.cinterop.ExperimentalForeignApi::class) 15 + private class IosExperimentLogger : ExperimentLogger { 16 + override val deviceLabel: String = 17 + "${UIDevice.currentDevice.systemName} ${UIDevice.currentDevice.model} ${UIDevice.currentDevice.systemVersion}" 18 + 19 + override fun write(filename: String, json: String): String? { 20 + val docs = (NSSearchPathForDirectoriesInDomains( 21 + NSDocumentDirectory, NSUserDomainMask, true 22 + ).firstOrNull() as? String) ?: return null 23 + val dir = "$docs/experiment_logs" 24 + NSFileManager.defaultManager.createDirectoryAtPath( 25 + dir, true, null, null 26 + ) 27 + val path = "$dir/$filename" 28 + val ok = (json as NSString).writeToFile( 29 + path = path, 30 + atomically = true, 31 + encoding = NSUTF8StringEncoding, 32 + error = null 33 + ) 34 + return if (ok) path else null 35 + } 9 36 } 10 37 11 38 @Composable 12 - actual fun rememberExperimentLogger(): ExperimentLogger = remember { IosExperimentLogger } 39 + actual fun rememberExperimentLogger(): ExperimentLogger = remember { IosExperimentLogger() }

+24

sample/composeApp/src/iosMain/kotlin/main.kt

··· 1 + import androidx.compose.runtime.CompositionLocalProvider 1 2 import androidx.compose.ui.window.ComposeUIViewController 2 3 import com.nate.posedetection.App 4 + import com.nate.posedetection.ExperimentLaunchSpec 5 + import com.nate.posedetection.LocalExperimentAutoSpec 3 6 import platform.UIKit.UIViewController 4 7 5 8 fun MainViewController(): UIViewController = ComposeUIViewController { App() } 9 + 10 + fun MainViewControllerWithAutoSpec( 11 + modelName: String?, 12 + startAtWallMs: Long, 13 + durationMs: Long, 14 + finishOnStop: Boolean, 15 + ): UIViewController { 16 + val spec = if (!modelName.isNullOrBlank() && durationMs > 0L) { 17 + ExperimentLaunchSpec( 18 + modelName = modelName, 19 + startAtWallMs = startAtWallMs, 20 + durationMs = durationMs, 21 + finishOnStop = finishOnStop, 22 + ) 23 + } else null 24 + return ComposeUIViewController { 25 + CompositionLocalProvider(LocalExperimentAutoSpec provides spec) { 26 + App() 27 + } 28 + } 29 + }

+43 -2

sample/iosApp/iosApp/iosApp.swift

··· 10 10 didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? 11 11 ) -> Bool { 12 12 window = UIWindow(frame: UIScreen.main.bounds) 13 - 13 + 14 + // Parse unattended experiment-mode launch args. 15 + // Accepts single-dash `-key value` pairs after a `--` separator (matches 16 + // `xcrun devicectl device process launch ... -- -key value` convention). 17 + let args = CommandLine.arguments 18 + var modelName: String? = nil 19 + var durationMs: Int64 = 0 20 + var startAtWallMs: Int64 = 0 21 + var finishOnStop: Bool = true 22 + var i = 0 23 + while i < args.count - 1 { 24 + let k = args[i] 25 + let v = args[i + 1] 26 + switch k { 27 + case "-test_model": 28 + modelName = v 29 + case "-test_duration_sec": 30 + if let secs = Int64(v) { durationMs = secs * 1000 } 31 + case "-test_duration_ms": 32 + durationMs = Int64(v) ?? 0 33 + case "-start_at_wall_ms": 34 + startAtWallMs = Int64(v) ?? 0 35 + case "-finish_on_stop": 36 + finishOnStop = (v == "1" || v.lowercased() == "true") 37 + default: break 38 + } 39 + i += 1 40 + } 41 + 42 + let rootVc: UIViewController 43 + if let name = modelName, durationMs > 0 { 44 + NSLog("[ExperimentAuto] launching with modelName=\(name) durationMs=\(durationMs) startAtWallMs=\(startAtWallMs)") 45 + rootVc = MainKt.MainViewControllerWithAutoSpec( 46 + modelName: name, 47 + startAtWallMs: startAtWallMs, 48 + durationMs: durationMs, 49 + finishOnStop: finishOnStop 50 + ) 51 + } else { 52 + rootVc = MainKt.MainViewController() 53 + } 54 + 14 55 if let window = window { 15 - window.rootViewController = MainKt.MainViewController() 56 + window.rootViewController = rootVc 16 57 window.makeKeyAndVisible() 17 58 } 18 59 return true

Configure Feed

Configure Feed