This repository has no description
0

Configure Feed

Select the types of activity you want to include in your feed.

Revert "chore: revert iOS pose to Vision (v4.15.1)"

This reverts commit 2df79f72f8d71ed17dcafa64c0b69fe5334663a1.

+406 -19
+84 -11
posedetection/build.gradle.kts
··· 4 4 5 5 mavenPublishing { 6 6 publishToMavenCentral(SonatypeHost.CENTRAL_PORTAL) 7 - coordinates("com.performancecoachlab.posedetection", "posedetection-compose", "4.15.1") 7 + coordinates("com.performancecoachlab.posedetection", "posedetection-compose", "4.15.0") 8 8 9 9 pom { 10 10 name.set("Pose Detection") ··· 62 62 baseName = "ComposeApp" 63 63 isStatic = true 64 64 } 65 + // MLKit pose detection is embedded via cinterop `staticLibraries` for 66 + // iosArm64 only — the library's klib ships the MLKit binaries so 67 + // downstream consumers get MLKit without running cocoapods themselves. 68 + // Sim targets fall back to Apple Vision (upstream MLKit has no 69 + // arm64-simulator slice). 70 + if (target.name == "iosArm64") { 71 + target.compilations.getByName("main").cinterops.create("mlkitAccurate") { 72 + defFile(layout.buildDirectory.file("mlkit-archives/mlkitAccurate.def").get().asFile) 73 + packageName("cocoapods.MLKitPoseDetectionAccurate") 74 + } 75 + } 65 76 } 66 77 67 78 sourceSets { ··· 121 132 } 122 133 123 134 // ============================================================================ 124 - // iOS pose detection 135 + // MLKit iOS integration 125 136 // ---------------------------------------------------------------------------- 126 - // iOS uses Apple Vision (VNDetectHumanBodyPoseRequest) for pose detection. 127 - // The MlKitPose expect/actual class is preserved across all iOS targets as 128 - // a stub returning isAvailable()=false, so FrameProcessor falls back to 129 - // Vision uniformly. MLKit infrastructure (sync-mlkit.sh, cinterop setup) 130 - // has been removed from v4.15.1 — shipping MLKit required consumer-side 131 - // iOS app setup (copy resource bundles, -ObjC linker flag, dynamic 132 - // framework) which violated the "zero extra steps" contract. A future 133 - // revival would need to ship MLKit as a pre-built XCFramework via SPM or 134 - // a Gradle plugin that auto-embeds it into the consumer's iOS app. 137 + // The library's iosArm64 klib ships MLKit statically-embedded via cinterop 138 + // `staticLibraries`. Downstream consumers get MLKit symbols without needing 139 + // cocoapods — standard Maven/Gradle KMP dependency resolution is enough. 140 + // 141 + // The `syncMlkitBinaries` task runs tools/sync-mlkit.sh, which fetches MLKit 142 + // pods into build/mlkit-staging and extracts per-target static archives into 143 + // build/mlkit-archives. `generateMlkitDefFile` then writes a cinterop .def 144 + // referencing those archives with absolute paths resolved at configuration 145 + // time. `cinteropMlkitAccurateIosArm64` is wired to depend on both. 135 146 // ============================================================================ 147 + 148 + val mlkitStagingDir = layout.buildDirectory.dir("mlkit-staging") 149 + val mlkitArchivesDir = layout.buildDirectory.dir("mlkit-archives") 150 + val mlkitDefFile = layout.buildDirectory.file("mlkit-archives/mlkitAccurate.def") 151 + 152 + val syncMlkitBinaries = tasks.register<Exec>("syncMlkitBinaries") { 153 + group = "mlkit" 154 + description = "Fetch MLKit pods + extract static archives for each Kotlin iOS target." 155 + inputs.file("tools/sync-mlkit.sh") 156 + outputs.dir(mlkitArchivesDir) 157 + executable = project.file("tools/sync-mlkit.sh").absolutePath 158 + environment("MLKIT_STAGING_DIR", mlkitStagingDir.get().asFile.absolutePath) 159 + environment("MLKIT_ARCHIVES_DIR", mlkitArchivesDir.get().asFile.absolutePath) 160 + // Only iosArm64 (iPhone device) is supported upstream; sim targets fall 161 + // back to Vision via the MlKitPose expect/actual stub. 162 + environment("MLKIT_TARGETS", "ios_arm64") 163 + } 164 + 165 + val generateMlkitDefFile = tasks.register("generateMlkitDefFile") { 166 + group = "mlkit" 167 + description = "Generate the MLKit cinterop .def with absolute archive paths." 168 + dependsOn(syncMlkitBinaries) 169 + outputs.file(mlkitDefFile) 170 + // Capture as locals so the closure doesn't hold project-level refs — 171 + // configuration-cache-safe. 172 + val pods = mlkitStagingDir.get().asFile.resolve("Pods").absolutePath 173 + val archives = mlkitArchivesDir.get().asFile.absolutePath 174 + val defFile = mlkitDefFile.get().asFile 175 + doLast { 176 + val defContent = buildString { 177 + appendLine("language = Objective-C") 178 + appendLine("modules = MLKitPoseDetectionAccurate MLKitPoseDetectionCommon MLKitVision") 179 + appendLine("package = cocoapods.MLKitPoseDetectionAccurate") 180 + appendLine( 181 + "compilerOpts = -fmodules " + 182 + "-F$pods/MLKitPoseDetectionAccurate/Frameworks " + 183 + "-F$pods/MLKitPoseDetectionCommon/Frameworks " + 184 + "-F$pods/MLKitVision/Frameworks " + 185 + "-F$pods/MLKitCommon/Frameworks " + 186 + "-F$pods/MLImage/Frameworks " + 187 + "-F$pods/MLKitXenoCommon/Frameworks" 188 + ) 189 + appendLine( 190 + "staticLibraries = " + 191 + "libMLKitPoseDetectionAccurate.a libMLKitPoseDetectionCommon.a " + 192 + "libMLKitVision.a libMLKitCommon.a libMLImage.a libMLKitXenoCommon.a " + 193 + "libGTMSessionFetcher.a libGoogleDataTransport.a libGoogleToolboxForMac.a " + 194 + "libGoogleUtilities.a libFBLPromises.a libnanopb.a" 195 + ) 196 + appendLine("libraryPaths.ios_arm64 = $archives/ios_arm64") 197 + appendLine( 198 + "linkerOpts = -ObjC -lc++ -lsqlite3 -lz " + 199 + "-framework Accelerate -framework CoreML" 200 + ) 201 + } 202 + defFile.writeText(defContent) 203 + } 204 + } 205 + 206 + tasks.matching { it.name.startsWith("cinteropMlkitAccurate") }.configureEach { 207 + dependsOn(generateMlkitDefFile) 208 + }
+176 -8
posedetection/src/iosArm64Main/kotlin/com/performancecoachlab/posedetection/camera/MlKitPose.kt
··· 1 1 package com.performancecoachlab.posedetection.camera 2 2 3 + import cocoapods.MLKitPoseDetectionAccurate.MLKAccuratePoseDetectorOptions 4 + import cocoapods.MLKitPoseDetectionAccurate.MLKCommonPoseDetectorOptions 5 + import cocoapods.MLKitPoseDetectionAccurate.MLKPose 6 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseDetector 7 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseDetectorModeStream 8 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkType 9 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeLeftAnkle 10 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeLeftElbow 11 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeLeftHeel 12 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeLeftHip 13 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeLeftIndexFinger 14 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeLeftKnee 15 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeLeftShoulder 16 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeLeftToe 17 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeLeftWrist 18 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeRightAnkle 19 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeRightElbow 20 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeRightHeel 21 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeRightHip 22 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeRightIndexFinger 23 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeRightKnee 24 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeRightShoulder 25 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeRightToe 26 + import cocoapods.MLKitPoseDetectionAccurate.MLKPoseLandmarkTypeRightWrist 27 + import cocoapods.MLKitPoseDetectionAccurate.MLKVisionImage 28 + import cocoapods.MLKitPoseDetectionAccurate.MLKVisionPoint 3 29 import com.performancecoachlab.posedetection.skeleton.Skeleton 4 30 import kotlinx.cinterop.ExperimentalForeignApi 31 + import kotlinx.cinterop.ObjCObjectVar 32 + import kotlinx.cinterop.alloc 33 + import kotlinx.cinterop.memScoped 34 + import kotlinx.cinterop.ptr 35 + import kotlinx.cinterop.useContents 36 + import platform.CoreGraphics.CGImageRelease 37 + import platform.CoreGraphics.CGRectMake 38 + import platform.CoreImage.CIContext 39 + import platform.CoreImage.CIImage 40 + import platform.CoreImage.createCGImage 5 41 import platform.CoreVideo.CVImageBufferRef 42 + import platform.Foundation.NSError 43 + import platform.UIKit.UIImage 6 44 7 - // iosArm64 stub: MLKit is not currently shipped via the library's published 8 - // artifacts (the resource bundles — ~9 MB of .tflite/.binarypb weights — 9 - // can't be flowed into a downstream app bundle from a KMP klib without 10 - // consumer-side setup). Reports unavailable so FrameProcessor falls back 11 - // to Apple Vision, same as the simulator targets. The infrastructure is 12 - // preserved for a future revival once MLKit distribution is figured out. 13 45 @OptIn(ExperimentalForeignApi::class) 14 46 internal actual class MlKitPose actual constructor() { 15 - actual fun isAvailable(): Boolean = false 47 + // CIContext is expensive; reuse across frames. 48 + private val ciContext: CIContext = CIContext.contextWithOptions(null) 49 + 50 + // Accurate-model, stream-mode detector — same config as Android. 51 + // Created lazily because the model loads on first use. 52 + private val detector: MLKPoseDetector by lazy { 53 + val options = MLKAccuratePoseDetectorOptions().apply { 54 + setDetectorMode(MLKPoseDetectorModeStream) 55 + } 56 + @Suppress("UNCHECKED_CAST") 57 + MLKPoseDetector.poseDetectorWithOptions( 58 + options as MLKCommonPoseDetectorOptions 59 + ) 60 + } 61 + 62 + actual fun isAvailable(): Boolean = true 16 63 17 64 actual fun detect( 18 65 buffer: CVImageBufferRef, ··· 25 72 cropWPx: Float, 26 73 cropHPx: Float, 27 74 timestamp: Long, 28 - ): Skeleton? = null 75 + ): Skeleton? { 76 + // Pre-rotate the pixels so the CGImage handed to MLKit is already 77 + // physically upright at oriented (logical/display) dimensions. MLKit 78 + // returns landmarks in the input CGImage's raw pixel space, so when 79 + // raw == oriented, landmarks land in the same oriented top-left pixel 80 + // space the rest of the library works in — no post-hoc remapping, 81 + // no UIImage/MLKVisionImage orientation juggling. 82 + val ciRaw = CIImage.imageWithCVPixelBuffer(buffer) ?: return null 83 + val ciOriented: CIImage = ciRaw.imageByApplyingOrientation(exifOrientation) 84 + 85 + // Rotated extent may have a non-zero origin depending on orientation. 86 + // Read the actual values so our render rect aligns with the pixels. 87 + var eOriginX = 0f 88 + var eOriginY = 0f 89 + var eW = 0f 90 + var eH = 0f 91 + ciOriented.extent.useContents { 92 + eOriginX = origin.x.toFloat() 93 + eOriginY = origin.y.toFloat() 94 + eW = size.width.toFloat() 95 + eH = size.height.toFloat() 96 + } 97 + 98 + // Compute the render rect in CIImage (bottom-left origin, absolute) 99 + // coords, anchored at the extent origin. 100 + val renderRect = if (useCrop) { 101 + // cropLeftPx / cropTopPx / cropWPx / cropHPx arrive in oriented 102 + // top-left space; flip Y into CIImage bottom-left space and shift 103 + // by the extent origin. 104 + CGRectMake( 105 + x = (eOriginX + cropLeftPx).toDouble(), 106 + y = (eOriginY + (eH - cropTopPx - cropHPx)).toDouble(), 107 + width = cropWPx.toDouble(), 108 + height = cropHPx.toDouble(), 109 + ) 110 + } else { 111 + CGRectMake( 112 + x = eOriginX.toDouble(), 113 + y = eOriginY.toDouble(), 114 + width = eW.toDouble(), 115 + height = eH.toDouble(), 116 + ) 117 + } 118 + val ciForRender = if (useCrop) ciOriented.imageByCroppingToRect(renderRect) else ciOriented 119 + val cgImage = ciContext.createCGImage(ciForRender, renderRect) ?: return null 120 + 121 + return try { 122 + // Plain UIImage with default .up orientation — we already rotated. 123 + val uiImage = UIImage(cgImage) 124 + val visionImage = MLKVisionImage(image = uiImage) 125 + // Leave visionImage.orientation at default .up. Setting it (or 126 + // giving the UIImage non-.up orientation metadata) either 127 + // double-rotates or leaves MLKit producing landmarks in a 128 + // different coord space from ours. 129 + 130 + val pose = memScoped { 131 + val errPtr = alloc<ObjCObjectVar<NSError?>>() 132 + @Suppress("UNCHECKED_CAST") 133 + val poses = detector.resultsInImage( 134 + visionImage as objcnames.protocols.MLKCompatibleImageProtocol, 135 + errPtr.ptr, 136 + ) as? List<MLKPose> 137 + poses?.firstOrNull() 138 + } ?: return null 139 + buildSkeletonFromPose( 140 + pose = pose, 141 + timestamp = timestamp, 142 + orientedW = orientedW, 143 + orientedH = orientedH, 144 + useCrop = useCrop, 145 + cropLeftPx = cropLeftPx, 146 + cropTopPx = cropTopPx, 147 + ) 148 + } finally { 149 + CGImageRelease(cgImage) 150 + } 151 + } 152 + 153 + private fun buildSkeletonFromPose( 154 + pose: MLKPose, 155 + timestamp: Long, 156 + orientedW: Float, 157 + orientedH: Float, 158 + useCrop: Boolean, 159 + cropLeftPx: Float, 160 + cropTopPx: Float, 161 + ): Skeleton { 162 + fun c(type: MLKPoseLandmarkType): Skeleton.SkeletonCoordinate? { 163 + val lm = pose.landmarkOfType(type) 164 + if (lm.inFrameLikelihood < LANDMARK_CONF_THRESHOLD) return null 165 + // MLKit returns pixel coords in the input image space. Because we 166 + // pre-rotated, that == oriented top-left space for MASK, or crop- 167 + // local for CROP. Add the crop offset to reach oriented full frame. 168 + val pos = lm.position as MLKVisionPoint 169 + val x = if (useCrop) cropLeftPx + pos.x.toFloat() else pos.x.toFloat() 170 + val y = if (useCrop) cropTopPx + pos.y.toFloat() else pos.y.toFloat() 171 + return Skeleton.SkeletonCoordinate(x, y) 172 + } 173 + return Skeleton( 174 + timestamp = timestamp, 175 + leftShoulder = c(MLKPoseLandmarkTypeLeftShoulder), 176 + rightShoulder = c(MLKPoseLandmarkTypeRightShoulder), 177 + leftElbow = c(MLKPoseLandmarkTypeLeftElbow), 178 + rightElbow = c(MLKPoseLandmarkTypeRightElbow), 179 + leftWrist = c(MLKPoseLandmarkTypeLeftWrist), 180 + rightWrist = c(MLKPoseLandmarkTypeRightWrist), 181 + leftHip = c(MLKPoseLandmarkTypeLeftHip), 182 + rightHip = c(MLKPoseLandmarkTypeRightHip), 183 + leftKnee = c(MLKPoseLandmarkTypeLeftKnee), 184 + rightKnee = c(MLKPoseLandmarkTypeRightKnee), 185 + leftAnkle = c(MLKPoseLandmarkTypeLeftAnkle), 186 + rightAnkle = c(MLKPoseLandmarkTypeRightAnkle), 187 + leftHeel = c(MLKPoseLandmarkTypeLeftHeel), 188 + rightHeel = c(MLKPoseLandmarkTypeRightHeel), 189 + leftToe = c(MLKPoseLandmarkTypeLeftToe), 190 + rightToe = c(MLKPoseLandmarkTypeRightToe), 191 + leftIndex = c(MLKPoseLandmarkTypeLeftIndexFinger), 192 + rightIndex = c(MLKPoseLandmarkTypeRightIndexFinger), 193 + height = orientedH, 194 + width = orientedW, 195 + ) 196 + } 29 197 }
+146
posedetection/tools/sync-mlkit.sh
··· 1 + #!/usr/bin/env bash 2 + # Build MLKit pose-detection static archives for all Kotlin/Native iOS targets 3 + # and stage them at build/mlkit-archives/<target>/lib<Pod>.a for cinterop to 4 + # embed into the library's klibs. 5 + # 6 + # Required env: MLKIT_STAGING_DIR, MLKIT_ARCHIVES_DIR 7 + # Optional env: MLKIT_TARGETS (space-separated; default "ios_arm64 ios_simulator_arm64 ios_x64") 8 + 9 + set -euo pipefail 10 + 11 + STAGING="${MLKIT_STAGING_DIR:?MLKIT_STAGING_DIR required}" 12 + ARCHIVES="${MLKIT_ARCHIVES_DIR:?MLKIT_ARCHIVES_DIR required}" 13 + TARGETS="${MLKIT_TARGETS:-ios_arm64 ios_simulator_arm64 ios_x64}" 14 + 15 + MLKIT_VERSION="1.0.0-beta16" 16 + IOS_DEPLOYMENT_TARGET="16.2" 17 + 18 + # Pods whose binaries we embed. Order matters for link-time resolution: 19 + # dependents before their deps. 20 + VENDORED_PODS=( 21 + MLKitPoseDetectionAccurate 22 + MLKitPoseDetectionCommon 23 + MLKitVision 24 + MLKitCommon 25 + MLImage 26 + MLKitXenoCommon 27 + ) 28 + # Built-from-source pods. Their framework binary paths inside the build dir 29 + # don't always match the pod name, so we map explicitly. 30 + # Format: "<pod-name>:<output-framework-name>" 31 + SOURCE_PODS=( 32 + "GTMSessionFetcher:GTMSessionFetcher" 33 + "GoogleDataTransport:GoogleDataTransport" 34 + "GoogleToolboxForMac:GoogleToolboxForMac" 35 + "GoogleUtilities:GoogleUtilities" 36 + "PromisesObjC:FBLPromises" 37 + "nanopb:nanopb" 38 + ) 39 + 40 + mkdir -p "$STAGING" 41 + mkdir -p "$ARCHIVES" 42 + cd "$STAGING" 43 + 44 + # Write a fresh Podfile for every run — ensures version changes propagate. 45 + cat > Podfile <<EOF 46 + platform :ios, '${IOS_DEPLOYMENT_TARGET}' 47 + use_frameworks! :linkage => :static 48 + 49 + install! 'cocoapods', :integrate_targets => false, :deterministic_uuids => false 50 + 51 + target 'MlkitSync' do 52 + pod 'MLKitPoseDetectionAccurate', '${MLKIT_VERSION}' 53 + end 54 + EOF 55 + 56 + echo "==> pod install in $STAGING" 57 + if [ ! -d Pods ] || [ ! -f Podfile.lock ] || ! diff -q Podfile Podfile.lock.input 2>/dev/null; then 58 + pod install --no-repo-update 59 + cp Podfile Podfile.lock.input 60 + fi 61 + 62 + # Map a Kotlin target name to an (sdk, arch) tuple. 63 + target_to_sdk() { 64 + case "$1" in 65 + ios_arm64) echo "iphoneos arm64" ;; 66 + ios_simulator_arm64) echo "iphonesimulator arm64" ;; 67 + ios_x64) echo "iphonesimulator x86_64" ;; 68 + *) echo "UNKNOWN UNKNOWN" ;; 69 + esac 70 + } 71 + 72 + extract_slice() { 73 + # $1 = input (fat or single-arch Mach-O) 74 + # $2 = desired arch (arm64, x86_64) 75 + # $3 = output path 76 + local input="$1" arch="$2" output="$3" 77 + mkdir -p "$(dirname "$output")" 78 + # lipo -thin fails on already-thin; use -info to branch 79 + if lipo -info "$input" 2>&1 | grep -q "Non-fat"; then 80 + local existing_arch 81 + existing_arch=$(lipo -info "$input" | sed -E 's/.*architecture: //') 82 + if [ "$existing_arch" = "$arch" ]; then 83 + cp "$input" "$output" 84 + else 85 + echo " skip: $input is $existing_arch, need $arch" 86 + return 1 87 + fi 88 + else 89 + lipo -thin "$arch" "$input" -output "$output" 2>/dev/null || { 90 + echo " skip: no $arch slice in $input" 91 + return 1 92 + } 93 + fi 94 + } 95 + 96 + for target in $TARGETS; do 97 + read -r sdk arch <<< "$(target_to_sdk "$target")" 98 + if [ "$sdk" = "UNKNOWN" ]; then 99 + echo "skipping unknown target $target" 100 + continue 101 + fi 102 + 103 + echo "==> Building source pods for $target ($sdk $arch)" 104 + # Build only the source pods — vendored ones don't need building. 105 + for entry in "${SOURCE_PODS[@]}"; do 106 + pod_name="${entry%%:*}" 107 + xcodebuild -project Pods/Pods.xcodeproj \ 108 + -target "$pod_name" \ 109 + -configuration Release \ 110 + -sdk "$sdk" \ 111 + -arch "$arch" \ 112 + ONLY_ACTIVE_ARCH=NO \ 113 + BUILD_LIBRARY_FOR_DISTRIBUTION=NO \ 114 + build 2>&1 | tail -3 115 + done 116 + 117 + out_dir="$ARCHIVES/$target" 118 + rm -rf "$out_dir" 119 + mkdir -p "$out_dir" 120 + 121 + echo "==> Extracting $target archives to $out_dir" 122 + # Vendored: pull the right slice from the fat .framework binary. 123 + for pod in "${VENDORED_PODS[@]}"; do 124 + fw_bin="Pods/$pod/Frameworks/$pod.framework/$pod" 125 + if [ -f "$fw_bin" ]; then 126 + extract_slice "$fw_bin" "$arch" "$out_dir/lib${pod}.a" || true 127 + else 128 + echo " miss: vendored $pod at $fw_bin" 129 + fi 130 + done 131 + # Built-from-source: single-arch output at build/Release-<sdk>/<pod>/<fw>.framework/<fw> 132 + for entry in "${SOURCE_PODS[@]}"; do 133 + pod_name="${entry%%:*}" 134 + fw_name="${entry##*:}" 135 + built="build/Release-$sdk/$pod_name/$fw_name.framework/$fw_name" 136 + if [ -f "$built" ]; then 137 + extract_slice "$built" "$arch" "$out_dir/lib${fw_name}.a" || true 138 + else 139 + echo " miss: built $pod_name at $built" 140 + fi 141 + done 142 + 143 + echo "==> $target: $(ls "$out_dir" | wc -l | tr -d ' ') archives" 144 + done 145 + 146 + echo "==> Done. Archives at $ARCHIVES"