This repository has no description
feat: improve io detection quality
author
nathan holland
date
(Jun 2, 2025, 9:46 PM +0300)
commit
dfd460ef
dfd460effc5be3a6b4768a7648ad819e82eed62a
parent
db617124
db6171243b9dd27b60d8acd8cf3f1ec6409636ff
+143
-33
+2
-2
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.android.kt
+2
-2
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.android.kt
···
13
13
import kotlinx.coroutines.suspendCancellableCoroutine
14
14
import kotlin.coroutines.resume
15
15
16
-
actual class InputFrame(val bitmap: Bitmap) {
16
+
actual class InputFrame(val bitmap: Bitmap, actual val timestamp: Long) {
17
17
actual fun toImageBitmap(): ImageBitmap {
18
18
return bitmap.asImageBitmap()
19
19
}
···
33
33
return suspendCancellableCoroutine { continuation ->
34
34
poseDetector.process(img)
35
35
.addOnSuccessListener { pose ->
36
-
val skeleton = skeleton(pose, System.currentTimeMillis(), img.width, img.height)
36
+
val skeleton = skeleton(pose, inputFrame.timestamp, img.width, img.height)
37
37
continuation.resume(skeleton)
38
38
}
39
39
.addOnFailureListener { e ->
+3
-3
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt
+3
-3
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt
···
5
5
import kotlinx.coroutines.withContext
6
6
7
7
actual suspend fun extractFrame(
8
-
videoPath: String, frame: Long
8
+
videoPath: String, frameTimestamp: Long
9
9
): InputFrame? {
10
10
return withContext(Dispatchers.IO) {
11
11
val retriever = MediaMetadataRetriever()
···
14
14
retriever.setDataSource(videoPath)
15
15
try {
16
16
val bitmap = retriever.getFrameAtTime(
17
-
frame * 1000L, // microseconds
17
+
frameTimestamp * 1000L, // microseconds
18
18
MediaMetadataRetriever.OPTION_CLOSEST
19
19
)
20
20
bitmap?.also {
21
-
return@withContext InputFrame(bitmap = it)
21
+
return@withContext InputFrame(bitmap = it, timestamp = frameTimestamp)
22
22
}
23
23
} catch (e: Exception) {
24
24
e.printStackTrace()
+1
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.kt
+1
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.kt
+1
-1
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt
+1
-1
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt
+128
-4
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
+128
-4
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/camera/FrameProcessor.kt
···
15
15
import platform.CoreGraphics.CGImageGetWidth
16
16
import platform.CoreGraphics.CGImageRef
17
17
import platform.CoreGraphics.CGPoint
18
+
import platform.CoreGraphics.CGRect
19
+
import platform.CoreGraphics.CGRectMake
18
20
import platform.Foundation.NSError
19
21
import platform.Vision.VNDetectHumanBodyPoseRequest
20
22
import platform.Vision.VNHumanBodyPoseObservation
···
35
37
import platform.Vision.VNImageRequestHandler
36
38
import platform.Vision.VNRecognizedPoint
37
39
import platform.Vision.VNRequest
38
-
import kotlin.native.runtime.NativeRuntimeApi
39
40
40
41
fun bodyPoseHandler(request: VNRequest): MutableMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint>? {
41
42
try {
···
50
51
51
52
@OptIn(ExperimentalForeignApi::class)
52
53
fun processObservation(observation: VNHumanBodyPoseObservation): MutableMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint> {
53
-
val points =
54
-
emptyMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint>().toMutableMap()
54
+
val points = emptyMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint>().toMutableMap()
55
55
observation.availableJointNames.forEach {
56
56
observation.recognizedPointForJointName(it as VNHumanBodyPoseObservationJointName, null)
57
57
?.also { point ->
···
76
76
}
77
77
}
78
78
79
+
@OptIn(ExperimentalForeignApi::class)
80
+
class FrameProcessor() {
81
+
var regionOfInterest = CGRectMake(0.0, 0.0, 1.0, 1.0)
82
+
@OptIn(BetaInteropApi::class)
83
+
fun analyseFrame(cgImage: CGImageRef?, timestamp: Long, onProccessed: (Skeleton?) -> Unit) {
84
+
autoreleasepool {
85
+
if (cgImage == null) {
86
+
onProccessed(null)
87
+
return
88
+
}
89
+
val width = CGImageGetWidth(cgImage)
90
+
val height = CGImageGetHeight(cgImage)
91
+
if (width.toUInt() == 0u || height.toUInt() == 0u) {
92
+
onProccessed(null)
93
+
return
94
+
}
95
+
memScoped {
96
+
val errorPtr = alloc<ObjCObjectVar<NSError?>>()
97
+
val requestHandler = VNImageRequestHandler(cgImage, mapOf<Any?, Any?>())
98
+
val request = VNDetectHumanBodyPoseRequest { request, error ->
99
+
if (error != null) {
100
+
onProccessed(null)
101
+
} else {
102
+
request?.also { vnRequest ->
103
+
val recognizedPoints = bodyPoseHandler(vnRequest)
104
+
regionOfInterest = calculateRegionOfInterest(recognizedPoints)
105
+
val updatedSkeleton = Skeleton(
106
+
timestamp = timestamp,
107
+
leftShoulder = recognizedPoints?.get(
108
+
VNHumanBodyPoseObservationJointNameLeftShoulder
109
+
)?.location?.toSkeletonPoint(width, height),
110
+
rightShoulder = recognizedPoints?.get(
111
+
VNHumanBodyPoseObservationJointNameRightShoulder
112
+
)?.location?.toSkeletonPoint(width, height),
113
+
leftElbow = recognizedPoints?.get(
114
+
VNHumanBodyPoseObservationJointNameLeftElbow
115
+
)?.location?.toSkeletonPoint(width, height),
116
+
rightElbow = recognizedPoints?.get(
117
+
VNHumanBodyPoseObservationJointNameRightElbow
118
+
)?.location?.toSkeletonPoint(width, height),
119
+
leftWrist = recognizedPoints?.get(
120
+
VNHumanBodyPoseObservationJointNameLeftWrist
121
+
)?.location?.toSkeletonPoint(width, height),
122
+
rightWrist = recognizedPoints?.get(
123
+
VNHumanBodyPoseObservationJointNameRightWrist
124
+
)?.location?.toSkeletonPoint(width, height),
125
+
leftHip = recognizedPoints?.get(
126
+
VNHumanBodyPoseObservationJointNameLeftHip
127
+
)?.location?.toSkeletonPoint(width, height),
128
+
rightHip = recognizedPoints?.get(
129
+
VNHumanBodyPoseObservationJointNameRightHip
130
+
)?.location?.toSkeletonPoint(width, height),
131
+
leftKnee = recognizedPoints?.get(
132
+
VNHumanBodyPoseObservationJointNameLeftKnee
133
+
)?.location?.toSkeletonPoint(width, height),
134
+
rightKnee = recognizedPoints?.get(
135
+
VNHumanBodyPoseObservationJointNameRightKnee
136
+
)?.location?.toSkeletonPoint(width, height),
137
+
leftAnkle = recognizedPoints?.get(
138
+
VNHumanBodyPoseObservationJointNameLeftAnkle
139
+
)?.location?.toSkeletonPoint(width, height),
140
+
rightAnkle = recognizedPoints?.get(
141
+
VNHumanBodyPoseObservationJointNameRightAnkle
142
+
)?.location?.toSkeletonPoint(width, height),
143
+
height = height.toFloat(),
144
+
width = width.toFloat()
145
+
)
146
+
onProccessed(updatedSkeleton)
147
+
}
148
+
}
149
+
}
150
+
request.regionOfInterest = regionOfInterest
151
+
try {
152
+
val result = runCatching {
153
+
request(requestHandler, request, errorPtr)
154
+
}
155
+
156
+
if (result.isFailure) {
157
+
println("Exception during performRequests: ${result.exceptionOrNull()?.message}")
158
+
onProccessed(null)
159
+
return@memScoped
160
+
}
161
+
162
+
if (errorPtr.value != null) {
163
+
println("Error performing request: ${errorPtr.value}")
164
+
onProccessed(null)
165
+
}
166
+
} catch (e: Throwable) {
167
+
println("Unable to perform the request: ${e.message}")
168
+
onProccessed(null)
169
+
}
170
+
}
171
+
}
172
+
}
173
+
}
174
+
79
175
@OptIn(ExperimentalForeignApi::class, BetaInteropApi::class)
80
-
fun processPose(cgImage: CGImageRef?, timestamp: Long, onProccessed: (Skeleton?) -> Unit) {
176
+
fun processPose(
177
+
cgImage: CGImageRef?,
178
+
timestamp: Long,
179
+
onProccessed: (Skeleton?) -> Unit
180
+
) {
81
181
autoreleasepool {
82
182
if (cgImage == null) {
83
183
onProccessed(null)
···
166
266
}
167
267
168
268
}
269
+
}
270
+
271
+
@OptIn(ExperimentalForeignApi::class)
272
+
fun calculateRegionOfInterest(
273
+
recognizedPoints: MutableMap<VNHumanBodyPoseObservationJointName, VNRecognizedPoint>?
274
+
): CValue<CGRect> {
275
+
val margin = 0.1
276
+
if (recognizedPoints.isNullOrEmpty()) {
277
+
return CGRectMake(0.0, 0.0, 1.0, 1.0) // Return full image area if no points are recognized
278
+
}
279
+
recognizedPoints.values.let { allPoints ->
280
+
val maxX = allPoints.minOf { it.x }
281
+
val minX = allPoints.minOf { it.x }
282
+
val maxY = allPoints.minOf { it.y }
283
+
val minY = allPoints.minOf { it.y }
284
+
285
+
return CGRectMake(
286
+
x = (minX - margin).coerceIn(0.0, 1.0),
287
+
y = (minY - margin).coerceIn(0.0, 1.0),
288
+
width = (maxX - minX + 2 * margin).coerceIn(0.0, 1.0),
289
+
height = (maxY - minY + 2 * margin).coerceIn(0.0, 1.0),
290
+
)
291
+
}
292
+
169
293
}
170
294
171
295
@OptIn(ExperimentalForeignApi::class)
+5
-3
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.ios.kt
+5
-3
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/InputFrame.ios.kt
···
2
2
3
3
import androidx.compose.ui.graphics.ImageBitmap
4
4
import androidx.compose.ui.graphics.Matrix
5
+
import com.performancecoachlab.posedetection.camera.FrameProcessor
5
6
import com.performancecoachlab.posedetection.camera.drawSkeleton
6
7
import com.performancecoachlab.posedetection.camera.processPose
7
8
import com.performancecoachlab.posedetection.camera.rotate180
···
21
22
import platform.UIKit.UIImageOrientation
22
23
import kotlin.coroutines.resume
23
24
24
-
actual class InputFrame @OptIn(ExperimentalForeignApi::class) constructor(val cgImage: CGImageRef) {
25
+
actual class InputFrame @OptIn(ExperimentalForeignApi::class) constructor(val cgImage: CGImageRef, actual val timestamp: Long) {
25
26
@OptIn(ExperimentalForeignApi::class)
26
27
actual fun toImageBitmap(): ImageBitmap {
27
28
return UIImage.imageWithCGImage(cgImage = cgImage).toImageBitmap()
···
33
34
}
34
35
35
36
36
-
actual class FrameAnalyser {
37
+
actual class FrameAnalyser actual constructor() {
38
+
private val frameProcessor = FrameProcessor()
37
39
@OptIn(ExperimentalForeignApi::class)
38
40
actual suspend fun analyseFrame(inputFrame: InputFrame): Skeleton? {
39
41
val img = inputFrame.cgImage
40
42
return suspendCancellableCoroutine { continuation ->
41
-
processPose(img, (NSDate().timeIntervalSince1970 * 1000).toLong(), onProccessed = {
43
+
frameProcessor.analyseFrame(img, inputFrame.timestamp, onProccessed = {
42
44
continuation.resume(it)
43
45
})
44
46
}
+3
-20
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt
+3
-20
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt
···
1
1
package com.performancecoachlab.posedetection.recording
2
2
3
-
import kotlinx.cinterop.CPointer
4
3
import kotlinx.cinterop.ExperimentalForeignApi
5
4
import kotlinx.coroutines.Dispatchers
6
5
import kotlinx.coroutines.IO
···
9
8
import platform.AVFoundation.AVAssetImageGenerator
10
9
import platform.AVFoundation.CMTimeValue
11
10
import platform.AVFoundation.valueWithCMTime
12
-
import platform.CoreGraphics.CGContextDrawImage
13
-
import platform.CoreGraphics.CGContextRotateCTM
14
-
import platform.CoreGraphics.CGImageGetHeight
15
-
import platform.CoreGraphics.CGImageGetWidth
16
-
import platform.CoreGraphics.CGImageRef
17
-
import platform.CoreGraphics.CGImageRelease
18
-
import platform.CoreGraphics.CGImageRetain
19
-
import platform.CoreGraphics.CGPointMake
20
-
import platform.CoreGraphics.CGRectMake
21
-
import platform.CoreGraphics.CGSizeMake
22
11
import platform.CoreMedia.CMTimeMake
23
12
import platform.CoreMedia.CMTimeMakeWithSeconds
24
13
import platform.Foundation.NSURL
25
14
import platform.Foundation.NSValue
26
-
import platform.UIKit.UIGraphicsBeginImageContext
27
-
import platform.UIKit.UIGraphicsEndImageContext
28
-
import platform.UIKit.UIGraphicsGetCurrentContext
29
-
import platform.UIKit.UIGraphicsGetImageFromCurrentImageContext
30
-
import platform.UIKit.UIImage
31
-
import platform.UIKit.UIImageOrientation
32
15
33
16
@OptIn(ExperimentalForeignApi::class)
34
17
actual suspend fun extractFrame(
35
18
videoPath: String,
36
-
frame: Long
19
+
frameTimestamp: Long
37
20
): InputFrame? {
38
21
return withContext(Dispatchers.IO) {
39
22
val asset = AVAsset.assetWithURL(createUrl(url = videoPath))
···
41
24
generator.appliesPreferredTrackTransform = true
42
25
generator.requestedTimeToleranceBefore = CMTimeMake(0, 600)
43
26
generator.requestedTimeToleranceAfter = CMTimeMake(0, 600)
44
-
val t = CMTimeMakeWithSeconds(frame.toDouble()/1000.0, preferredTimescale = 600)
27
+
val t = CMTimeMakeWithSeconds(frameTimestamp.toDouble()/1000.0, preferredTimescale = 600)
45
28
val time = NSValue.valueWithCMTime(t)
46
29
try {
47
30
val cgImage =
48
31
generator.copyCGImageAtTime(time.CMTimeValue, actualTime = null, error = null)
49
32
cgImage?.let {
50
-
return@withContext InputFrame(it)
33
+
return@withContext InputFrame(cgImage = it, timestamp = frameTimestamp)
51
34
}
52
35
} catch (e: Exception) {
53
36
e.printStackTrace()