This repository has no description
feat: retrieve timestamps from video
author
nathan holland
committer
florian-kima
date
(Sep 17, 2025, 1:21 PM +0300)
commit
a49c31c2
a49c31c2042cbb34e8aa2f909c6c7f90bbfba5ef
parent
cb909961
cb9099616643eaca0be111fd1f28426ce3a744ab
+198
-2
+46
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt
+46
posedetection/src/androidMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.android.kt
···
2
2
3
3
import android.app.Application
4
4
import android.content.Context
5
+
import android.media.MediaExtractor
6
+
import android.media.MediaFormat
5
7
import android.media.MediaMetadataRetriever
6
8
import kotlinx.coroutines.Dispatchers
7
9
import kotlinx.coroutines.withContext
···
37
39
}
38
40
return@withContext null
39
41
}
42
+
}
43
+
44
+
actual suspend fun listVideoFrameTimestamps(
45
+
videoPath: String,
46
+
): List<Long> = withContext(Dispatchers.IO) {
47
+
val ctx = VideoExtractionContext.get()
48
+
val uri = videoPath.toUri()
49
+
val extractor = MediaExtractor()
50
+
51
+
val out = ArrayList<Long>()
52
+
try {
53
+
extractor.setDataSource(ctx, uri, null)
54
+
55
+
// Select video track
56
+
var videoTrack = -1
57
+
for (i in 0 until extractor.trackCount) {
58
+
val format = extractor.getTrackFormat(i)
59
+
val mime = format.getString(MediaFormat.KEY_MIME) ?: continue
60
+
if (mime.startsWith("video/")) {
61
+
videoTrack = i
62
+
break
63
+
}
64
+
}
65
+
if (videoTrack == -1) return@withContext out
66
+
67
+
extractor.selectTrack(videoTrack)
68
+
69
+
var idx = 0
70
+
while (true) {
71
+
val sampleTimeUs = extractor.sampleTime
72
+
if (sampleTimeUs < 0) break
73
+
74
+
out.add(sampleTimeUs / 1000L) // ms
75
+
if (out.size >= Int.MAX_VALUE) break
76
+
idx++
77
+
78
+
if (!extractor.advance()) break
79
+
}
80
+
} catch (_: Throwable) {
81
+
// swallow; return what we have
82
+
} finally {
83
+
extractor.release()
84
+
}
85
+
out
40
86
}
41
87
42
88
actual object VideoExtractionContext {
+4
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt
+4
posedetection/src/commonMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.kt
+141
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt
+141
-1
posedetection/src/iosMain/kotlin/com/performancecoachlab/posedetection/recording/VideoUtils.ios.kt
···
1
1
package com.performancecoachlab.posedetection.recording
2
2
3
+
import kotlinx.cinterop.BetaInteropApi
3
4
import kotlinx.cinterop.ExperimentalForeignApi
5
+
import kotlinx.cinterop.ObjCObjectVar
6
+
import kotlinx.cinterop.alloc
7
+
import kotlinx.cinterop.memScoped
8
+
import kotlinx.cinterop.ptr
9
+
import kotlinx.cinterop.value
4
10
import kotlinx.coroutines.Dispatchers
5
11
import kotlinx.coroutines.IO
6
12
import kotlinx.coroutines.withContext
7
13
import platform.AVFoundation.AVAsset
8
14
import platform.AVFoundation.AVAssetImageGenerator
15
+
import platform.AVFoundation.AVAssetReader
16
+
import platform.AVFoundation.AVAssetReaderTrackOutput
17
+
import platform.AVFoundation.AVAssetTrack
18
+
import platform.AVFoundation.AVMediaTypeVideo
9
19
import platform.AVFoundation.CMTimeValue
20
+
import platform.AVFoundation.tracksWithMediaType
10
21
import platform.AVFoundation.valueWithCMTime
11
22
import platform.CoreMedia.CMTimeMake
12
23
import platform.CoreMedia.CMTimeMakeWithSeconds
24
+
import platform.CoreMedia.CMTimeGetSeconds
25
+
import platform.CoreMedia.CMSampleBufferGetPresentationTimeStamp
26
+
import platform.CoreFoundation.CFRelease
27
+
import platform.Foundation.NSError
13
28
import platform.Foundation.NSURL
14
29
import platform.Foundation.NSValue
30
+
import platform.Foundation.NSFileManager
15
31
16
32
@OptIn(ExperimentalForeignApi::class)
17
33
actual suspend fun extractFrame(
···
39
55
}
40
56
}
41
57
58
+
@OptIn(ExperimentalForeignApi::class, BetaInteropApi::class)
59
+
actual suspend fun listVideoFrameTimestamps(
60
+
videoPath: String,
61
+
): List<Long> = withContext(Dispatchers.IO) {
62
+
val out = mutableListOf<Long>()
63
+
64
+
try {
65
+
val resolvedUrl = createUrl(url = videoPath)
66
+
println("[iOS] listVideoFrameTimestamps using URL: ${'$'}{resolvedUrl.absoluteString}")
67
+
68
+
// AVAssetReader requires a local file URL. Bail out early for remote URLs.
69
+
if (!resolvedUrl.isFileURL()) {
70
+
println("AVAssetReader requires a local file URL; got: ${'$'}{resolvedUrl.scheme}:// ...")
71
+
return@withContext out
72
+
}
73
+
74
+
// Verify the file actually exists at path
75
+
val path = resolvedUrl.path
76
+
if (path == null || !NSFileManager.defaultManager.fileExistsAtPath(path)) {
77
+
println("File does not exist at path: ${'$'}path")
78
+
return@withContext out
79
+
}
80
+
81
+
val asset = AVAsset.assetWithURL(resolvedUrl)
82
+
83
+
// Use synchronous approach - get tracks directly using tracksWithMediaType
84
+
val videoTracks = asset.tracksWithMediaType(AVMediaTypeVideo)
85
+
86
+
if (videoTracks.isEmpty()) {
87
+
println("No video tracks found in video: ${'$'}videoPath")
88
+
return@withContext out
89
+
}
90
+
91
+
val videoTrack = videoTracks.first() as AVAssetTrack
92
+
println("Found video track: ${'$'}{videoTrack}")
93
+
94
+
memScoped {
95
+
val errorPtr = alloc<ObjCObjectVar<NSError?>>()
96
+
val reader = AVAssetReader.assetReaderWithAsset(asset, error = errorPtr.ptr)
97
+
98
+
if (reader == null) {
99
+
val err = errorPtr.value
100
+
println("Failed to create AVAssetReader: ${'$'}{err?.localizedDescription}")
101
+
return@withContext out
102
+
}
103
+
104
+
val trackOutput = AVAssetReaderTrackOutput.assetReaderTrackOutputWithTrack(
105
+
track = videoTrack,
106
+
outputSettings = null
107
+
)
108
+
109
+
if (!reader.canAddOutput(trackOutput)) {
110
+
println("Cannot add track output to reader")
111
+
return@withContext out
112
+
}
113
+
114
+
reader.addOutput(trackOutput)
115
+
116
+
if (!reader.startReading()) {
117
+
println("Failed to start reading: ${'$'}{reader.error?.localizedDescription}")
118
+
return@withContext out
119
+
}
120
+
121
+
println("Started reading video frames...")
122
+
var frameCount = 0
123
+
while (true) {
124
+
val sampleBuffer = trackOutput.copyNextSampleBuffer()
125
+
if (sampleBuffer == null) {
126
+
break
127
+
}
128
+
129
+
val presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
130
+
val timeInSeconds = CMTimeGetSeconds(presentationTime)
131
+
val timeInMillis = (timeInSeconds * 1000.0).toLong()
132
+
133
+
out.add(timeInMillis)
134
+
frameCount++
135
+
136
+
// Release the sample buffer
137
+
CFRelease(sampleBuffer)
138
+
139
+
if (out.size >= Int.MAX_VALUE) break
140
+
}
141
+
142
+
println("Extracted ${'$'}frameCount frame timestamps")
143
+
}
144
+
145
+
} catch (e: Exception) {
146
+
println("Exception in listVideoFrameTimestamps: ${'$'}{e.message}")
147
+
e.printStackTrace()
148
+
}
149
+
150
+
println("Returning ${'$'}{out.size} timestamps")
151
+
out
152
+
}
153
+
42
154
fun createUrl(url: String): NSURL {
43
-
return NSURL.fileURLWithPath(url)
155
+
val trimmed = url.trim()
156
+
157
+
// Remote URLs: return as-is
158
+
if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) {
159
+
return requireNotNull(NSURL.URLWithString(trimmed)) { "Invalid http(s) URL: ${'$'}trimmed" }
160
+
}
161
+
162
+
// Well-formed file URLs
163
+
if (trimmed.startsWith("file://")) {
164
+
// Normalize to a proper file URL via its path component to avoid odd host parts
165
+
NSURL.URLWithString(trimmed)?.let { fileUrl ->
166
+
fileUrl.path?.let { path ->
167
+
return NSURL.fileURLWithPath(path)
168
+
}
169
+
return fileUrl
170
+
}
171
+
}
172
+
173
+
// Malformed file: prefix (e.g., "file:/var/..."), coerce to path
174
+
if (trimmed.startsWith("file:")) {
175
+
var path = trimmed.removePrefix("file:")
176
+
// Remove duplicate leading slashes
177
+
while (path.startsWith("//")) path = path.removePrefix("//")
178
+
if (!path.startsWith("/")) path = "/${'$'}path"
179
+
return NSURL.fileURLWithPath(path)
180
+
}
181
+
182
+
// Otherwise treat as a filesystem path
183
+
return NSURL.fileURLWithPath(trimmed)
44
184
}
45
185
46
186
actual object VideoExtractionContext
+7
-1
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
+7
-1
sample/composeApp/src/commonMain/kotlin/com/nate/posedetection/App.kt
···
52
52
import com.performancecoachlab.posedetection.recording.InputFrame
53
53
import com.performancecoachlab.posedetection.recording.Label
54
54
import com.performancecoachlab.posedetection.recording.extractFrame
55
+
import com.performancecoachlab.posedetection.recording.listVideoFrameTimestamps
55
56
import com.performancecoachlab.posedetection.skeleton.Pose
56
57
import com.performancecoachlab.posedetection.skeleton.SkeletonRepository
57
58
import io.github.vinceglb.filekit.FileKit
···
71
72
72
73
@Composable
73
74
internal fun App() = AppTheme {
74
-
var selectedTabIndex by remember { mutableStateOf(0) }
75
+
var selectedTabIndex by remember { mutableStateOf(1) }
75
76
val tabs = listOf("Camera Feed", "Recorded Video")
76
77
Column {
77
78
TabRow(selectedTabIndex = selectedTabIndex) {
···
187
188
e.printStackTrace()
188
189
}
189
190
}
191
+
}
192
+
193
+
LaunchedEffect(url){
194
+
val timestamps = listVideoFrameTimestamps(url)
195
+
println("Timestamps: $timestamps")
190
196
}
191
197
192
198
LaunchedEffect(url, frame) {