This repository has no description
feat: PoseFocusMode { MASK, CROP } for focus-area pose input
Adds a new parameter on the CameraView composable so callers can choose
how the focus rectangle is applied to the pose input. MASK preserves
existing behaviour — black out non-focus region and downscale the full
frame. CROP geometrically restricts the pose input to just the focus
rectangle before downscaling, giving the MLKit model more effective
pixels on the subject at the same downscaled side length; landmarks are
returned in full-frame coordinates via an offset that flows from
buildMlKitPoseInput through skeletonFromPoseScaled.
Object detection is intentionally unaffected — YOLO always sees the
full, unmasked frame.
Sample app exposes a menu toggle (Mask ↔ Crop) and points the demo at
the left half of the frame to exercise both modes.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
author
virtualintern
co-author
Claude Opus 4.7 (1M context)
date
(Apr 20, 2026, 6:34 PM +0300)
commit
b7c63078
b7c63078b2c27c10084c9121f15fbae5d3a04119
parent
a538c4c3
a538c4c3b4cfeb0493c2b333fc13bbf7c5669657
+131
-29
+8
-1
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
+8
-1
posedetection/src/androidMain/kotlin/com.performancecoachlab/posedetection/camera/CameraView.android.kt
···
119
119
previewFillMode: PreviewFillMode,
120
120
recordingId: String?,
121
121
focusArea: Rect?,
122
+
poseFocusMode: PoseFocusMode,
122
123
controller: CameraViewController?,
123
124
onRecordToggled: (Boolean) -> Unit,
124
125
onVideoSaved: (String, String) -> Unit,
···
148
149
val executor = remember { Executors.newSingleThreadExecutor() }
149
150
150
151
var focus by remember { mutableStateOf(focusArea) }
152
+
var currentPoseFocusMode by remember { mutableStateOf(poseFocusMode) }
151
153
var objectDetector by remember { mutableStateOf(objectModel) }
152
154
var currentDetectMode by remember { mutableStateOf(detectMode) }
153
155
···
161
163
// Update focus when focusArea changes
162
164
LaunchedEffect(focusArea) {
163
165
focus = focusArea
166
+
}
167
+
168
+
LaunchedEffect(poseFocusMode) {
169
+
currentPoseFocusMode = poseFocusMode
164
170
}
165
171
166
172
// Update object model when it changes
···
339
345
objectClient,
340
346
poseClient,
341
347
now,
342
-
area
348
+
area,
349
+
currentPoseFocusMode,
343
350
) { analysisResult, frameBitmap ->
344
351
try {
345
352
// Only update repositories/results for detectors that actually ran.
+82
-27
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.android.kt
+82
-27
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/camera/Utils.android.kt
···
227
227
poseDetector: PoseDetector?,
228
228
timestamp: Long,
229
229
focusArea: Rect?,
230
+
poseFocusMode: PoseFocusMode = PoseFocusMode.MASK,
230
231
onComplete: (AnalysisResult, Bitmap) -> Unit
231
232
) {
232
233
// If both are null, still close to avoid stalling the camera pipeline.
···
250
251
buildMlKitPoseInput(
251
252
analysisBitmap = analysisBitmap,
252
253
focusArea = focusArea,
254
+
poseFocusMode = poseFocusMode,
253
255
downscaleMaxSidePx = 256
254
256
)
255
257
}
···
282
284
onComplete = onComplete,
283
285
mlKitScaleX = mlKitPoseInput?.scaleX ?: 1f,
284
286
mlKitScaleY = mlKitPoseInput?.scaleY ?: 1f,
287
+
mlKitOffsetX = mlKitPoseInput?.offsetX ?: 0f,
288
+
mlKitOffsetY = mlKitPoseInput?.offsetY ?: 0f,
285
289
)
286
290
}
287
291
···
301
305
onComplete: (AnalysisResult, Bitmap) -> Unit,
302
306
mlKitScaleX: Float = 1f,
303
307
mlKitScaleY: Float = 1f,
308
+
mlKitOffsetX: Float = 0f,
309
+
mlKitOffsetY: Float = 0f,
304
310
) {
305
311
// Launch pose detection on a separate thread so it runs in parallel with object detection.
306
312
val poseFuture = if (poseDetector != null && mlKitImage != null) {
···
314
320
height = height,
315
321
scaleX = mlKitScaleX,
316
322
scaleY = mlKitScaleY,
323
+
offsetX = mlKitOffsetX,
324
+
offsetY = mlKitOffsetY,
317
325
)
318
326
}.onFailure { t ->
319
327
Logger.e(t) { "MLKit poseDetector.process failed" }
···
707
715
val image: InputImage,
708
716
val scaleX: Float,
709
717
val scaleY: Float,
718
+
val offsetX: Float = 0f,
719
+
val offsetY: Float = 0f,
710
720
)
711
721
712
722
private fun buildMlKitPoseInput(
713
723
analysisBitmap: Bitmap,
714
724
focusArea: Rect?,
725
+
poseFocusMode: PoseFocusMode,
715
726
downscaleMaxSidePx: Int = 360,
716
727
): MlKitPoseInput {
717
-
// 1) Apply optional focus mask in full-resolution upright coordinates.
718
-
val poseBitmapFull = if (focusArea == null) {
719
-
analysisBitmap
720
-
} else {
721
-
val out = PoseMaskBitmapPool.obtain(
722
-
analysisBitmap.width,
723
-
analysisBitmap.height,
724
-
analysisBitmap.config ?: Bitmap.Config.ARGB_8888
725
-
)
726
-
Canvas(out).drawBitmap(analysisBitmap, 0f, 0f, null)
727
-
out.applyFocusAreaMaskInPlace(focusArea = focusArea, angle = 0)
728
-
out
728
+
// 1) Build the pose source bitmap — mask (black out non-focus region) or
729
+
// crop (geometrically restrict to focus region). Crop gives the model
730
+
// higher effective resolution of the focus area at the same downscale.
731
+
//
732
+
// Crop-mode also records offsetX/offsetY in full-bitmap pixel coords so
733
+
// landmarks (which come back in crop-local coords) can be remapped.
734
+
val useCrop = focusArea != null && poseFocusMode == PoseFocusMode.CROP
735
+
736
+
val offsetX: Float
737
+
val offsetY: Float
738
+
val poseBitmapFull: Bitmap = when {
739
+
focusArea == null -> {
740
+
offsetX = 0f
741
+
offsetY = 0f
742
+
analysisBitmap
743
+
}
744
+
useCrop -> {
745
+
val w = analysisBitmap.width
746
+
val h = analysisBitmap.height
747
+
val leftPx = (focusArea.left * w).toInt().coerceIn(0, w)
748
+
val topPx = (focusArea.top * h).toInt().coerceIn(0, h)
749
+
val rightPx = (focusArea.right * w).toInt().coerceIn(leftPx, w)
750
+
val bottomPx = (focusArea.bottom * h).toInt().coerceIn(topPx, h)
751
+
val cropW = rightPx - leftPx
752
+
val cropH = bottomPx - topPx
753
+
if (cropW <= 0 || cropH <= 0) {
754
+
// Degenerate focus rect — fall back to full frame untouched.
755
+
offsetX = 0f
756
+
offsetY = 0f
757
+
analysisBitmap
758
+
} else {
759
+
offsetX = leftPx.toFloat()
760
+
offsetY = topPx.toFloat()
761
+
Bitmap.createBitmap(analysisBitmap, leftPx, topPx, cropW, cropH)
762
+
}
763
+
}
764
+
else -> {
765
+
offsetX = 0f
766
+
offsetY = 0f
767
+
val out = PoseMaskBitmapPool.obtain(
768
+
analysisBitmap.width,
769
+
analysisBitmap.height,
770
+
analysisBitmap.config ?: Bitmap.Config.ARGB_8888
771
+
)
772
+
Canvas(out).drawBitmap(analysisBitmap, 0f, 0f, null)
773
+
out.applyFocusAreaMaskInPlace(focusArea = focusArea, angle = 0)
774
+
out
775
+
}
729
776
}
730
777
731
778
// 2) Downscale for MLKit (pose is generally robust at lower res).
···
738
785
image = InputImage.fromBitmap(poseBitmapFull, 0),
739
786
scaleX = 1f,
740
787
scaleY = 1f,
788
+
offsetX = offsetX,
789
+
offsetY = offsetY,
741
790
)
742
791
}
743
792
···
756
805
image = InputImage.fromBitmap(downscaled, 0),
757
806
scaleX = scaleBackX,
758
807
scaleY = scaleBackY,
808
+
offsetX = offsetX,
809
+
offsetY = offsetY,
759
810
)
760
811
}
761
812
762
813
private fun PoseLandmark?.toSkeletonCoordsScaled(
763
814
scaleX: Float,
764
-
scaleY: Float
815
+
scaleY: Float,
816
+
offsetX: Float,
817
+
offsetY: Float,
765
818
): Skeleton.SkeletonCoordinate? {
766
819
val pos = this?.position ?: return null
767
820
return Skeleton.SkeletonCoordinate(
768
-
x = pos.x * scaleX,
769
-
y = pos.y * scaleY,
821
+
x = pos.x * scaleX + offsetX,
822
+
y = pos.y * scaleY + offsetY,
770
823
)
771
824
}
772
825
···
777
830
height: Int,
778
831
scaleX: Float,
779
832
scaleY: Float,
833
+
offsetX: Float = 0f,
834
+
offsetY: Float = 0f,
780
835
): Skeleton {
781
836
return Skeleton(
782
837
timestamp = timestamp,
783
838
leftShoulder = pose?.getPoseLandmark(PoseLandmark.LEFT_SHOULDER)
784
-
?.toSkeletonCoordsScaled(scaleX, scaleY),
839
+
?.toSkeletonCoordsScaled(scaleX, scaleY, offsetX, offsetY),
785
840
rightShoulder = pose?.getPoseLandmark(PoseLandmark.RIGHT_SHOULDER)
786
-
?.toSkeletonCoordsScaled(scaleX, scaleY),
841
+
?.toSkeletonCoordsScaled(scaleX, scaleY, offsetX, offsetY),
787
842
leftElbow = pose?.getPoseLandmark(PoseLandmark.LEFT_ELBOW)
788
-
?.toSkeletonCoordsScaled(scaleX, scaleY),
843
+
?.toSkeletonCoordsScaled(scaleX, scaleY, offsetX, offsetY),
789
844
rightElbow = pose?.getPoseLandmark(PoseLandmark.RIGHT_ELBOW)
790
-
?.toSkeletonCoordsScaled(scaleX, scaleY),
845
+
?.toSkeletonCoordsScaled(scaleX, scaleY, offsetX, offsetY),
791
846
leftWrist = pose?.getPoseLandmark(PoseLandmark.LEFT_WRIST)
792
-
?.toSkeletonCoordsScaled(scaleX, scaleY),
847
+
?.toSkeletonCoordsScaled(scaleX, scaleY, offsetX, offsetY),
793
848
rightWrist = pose?.getPoseLandmark(PoseLandmark.RIGHT_WRIST)
794
-
?.toSkeletonCoordsScaled(scaleX, scaleY),
849
+
?.toSkeletonCoordsScaled(scaleX, scaleY, offsetX, offsetY),
795
850
leftHip = pose?.getPoseLandmark(PoseLandmark.LEFT_HIP)
796
-
?.toSkeletonCoordsScaled(scaleX, scaleY),
851
+
?.toSkeletonCoordsScaled(scaleX, scaleY, offsetX, offsetY),
797
852
rightHip = pose?.getPoseLandmark(PoseLandmark.RIGHT_HIP)
798
-
?.toSkeletonCoordsScaled(scaleX, scaleY),
853
+
?.toSkeletonCoordsScaled(scaleX, scaleY, offsetX, offsetY),
799
854
leftKnee = pose?.getPoseLandmark(PoseLandmark.LEFT_KNEE)
800
-
?.toSkeletonCoordsScaled(scaleX, scaleY),
855
+
?.toSkeletonCoordsScaled(scaleX, scaleY, offsetX, offsetY),
801
856
rightKnee = pose?.getPoseLandmark(PoseLandmark.RIGHT_KNEE)
802
-
?.toSkeletonCoordsScaled(scaleX, scaleY),
857
+
?.toSkeletonCoordsScaled(scaleX, scaleY, offsetX, offsetY),
803
858
leftAnkle = pose?.getPoseLandmark(PoseLandmark.LEFT_ANKLE)
804
-
?.toSkeletonCoordsScaled(scaleX, scaleY),
859
+
?.toSkeletonCoordsScaled(scaleX, scaleY, offsetX, offsetY),
805
860
rightAnkle = pose?.getPoseLandmark(PoseLandmark.RIGHT_ANKLE)
806
-
?.toSkeletonCoordsScaled(scaleX, scaleY),
861
+
?.toSkeletonCoordsScaled(scaleX, scaleY, offsetX, offsetY),
807
862
width = width.toFloat(),
808
863
height = height.toFloat(),
809
864
)
+13
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.kt
+13
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.kt
···
30
30
FILL
31
31
}
32
32
33
+
/**
34
+
* How the focus area is applied to the pose input.
35
+
*
36
+
* MASK — black out everything outside the focus area, then downscale the full
37
+
* frame. Object detection is unaffected (it always sees the unmasked frame).
38
+
*
39
+
* CROP — crop the frame to the focus area, then downscale only the crop. Gives
40
+
* the pose model higher effective resolution of the focus region at the same
41
+
* downscaled side length. Landmarks are remapped back to full-frame coords.
42
+
*/
43
+
enum class PoseFocusMode { MASK, CROP }
44
+
33
45
@Composable
34
46
expect fun CameraView(
35
47
skeletonRepository: SkeletonRepository,
···
46
58
previewFillMode: PreviewFillMode = PreviewFillMode.FILL,
47
59
recordingId: String? = null,
48
60
focusArea: Rect? = null,
61
+
poseFocusMode: PoseFocusMode = PoseFocusMode.MASK,
49
62
controller: CameraViewController? = null,
50
63
onRecordToggled: (Boolean) -> Unit = {},
51
64
onVideoSaved: (String, String) -> Unit
+1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt
+1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/CameraView.ios.kt
+27
-1
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
+27
-1
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
···
65
65
import com.performancecoachlab.posedetection.camera.DetectMode
66
66
import com.performancecoachlab.posedetection.camera.DrawableObject
67
67
import com.performancecoachlab.posedetection.camera.DrawableShape
68
+
import com.performancecoachlab.posedetection.camera.PoseFocusMode
68
69
import com.performancecoachlab.posedetection.camera.PreviewFillMode
69
70
import com.performancecoachlab.posedetection.custom.CustomObjectRespository
70
71
import com.performancecoachlab.posedetection.custom.ModelPath
···
416
417
var previewFillMode by remember { mutableStateOf(PreviewFillMode.FIT) }
417
418
var menuExpanded by remember { mutableStateOf(false) }
418
419
var detectMode by remember { mutableStateOf(DetectMode.BOTH) }
420
+
var poseFocusMode by remember { mutableStateOf(PoseFocusMode.MASK) }
419
421
val controller = remember { CameraViewControllerImpl() }
420
422
421
423
// Experiment Mode state — buffers per-detection events into a JSON log
···
589
591
},
590
592
objectModel = generalModel,
591
593
modifier = Modifier.weight(1f),
592
-
focusArea = Rect(0f, 0f, 1f, 1f),
594
+
focusArea = Rect(0f, 0f, 0.5f, 1f),
595
+
poseFocusMode = poseFocusMode,
593
596
frontCamera = frontCamera,
594
597
useUltraWide = ultrawide,
595
598
previewFillMode = previewFillMode,
···
648
651
onClick = {
649
652
previewFillMode =
650
653
if (previewFillMode == PreviewFillMode.FIT) PreviewFillMode.FILL else PreviewFillMode.FIT
654
+
}
655
+
)
656
+
657
+
// Pose focus mode toggle (mask vs crop) — only affects
658
+
// the pose input; object detection always sees the full frame.
659
+
DropdownMenuItem(
660
+
text = {
661
+
Row(
662
+
horizontalArrangement = Arrangement.SpaceBetween,
663
+
verticalAlignment = Alignment.CenterVertically,
664
+
modifier = Modifier.fillMaxWidth()
665
+
) {
666
+
Text(text = if (poseFocusMode == PoseFocusMode.CROP) "Pose: Crop focus" else "Pose: Mask focus")
667
+
Spacer(Modifier.width(12.dp))
668
+
Switch(
669
+
checked = poseFocusMode == PoseFocusMode.CROP,
670
+
onCheckedChange = null
671
+
)
672
+
}
673
+
},
674
+
onClick = {
675
+
poseFocusMode =
676
+
if (poseFocusMode == PoseFocusMode.MASK) PoseFocusMode.CROP else PoseFocusMode.MASK
651
677
}
652
678
)
653
679