✨ Use ffprobe for .wav markers extraction · gwen.works/shapemaker@889dcc3

+14 -141

Cargo.lock

··· 1218 1218 ] 1219 1219 1220 1220 [[package]] 1221 - name = "extended" 1222 - version = "0.1.0" 1223 - source = "registry+https://github.com/rust-lang/crates.io-index" 1224 - checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" 1225 - 1226 - [[package]] 1227 1221 name = "fake-simd" 1228 1222 version = "0.1.2" 1229 1223 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4291 4285 "itertools 0.14.0", 4292 4286 "pico-args", 4293 4287 "rand 0.9.2", 4288 + "serde-aux", 4294 4289 "shapemaker", 4295 4290 "tokio", 4296 4291 ] ··· 4404 4399 ] 4405 4400 4406 4401 [[package]] 4402 + name = "serde-aux" 4403 + version = "4.7.0" 4404 + source = "registry+https://github.com/rust-lang/crates.io-index" 4405 + checksum = "207f67b28fe90fb596503a9bf0bf1ea5e831e21307658e177c5dfcdfc3ab8a0a" 4406 + dependencies = [ 4407 + "chrono", 4408 + "serde", 4409 + "serde-value", 4410 + "serde_json", 4411 + ] 4412 + 4413 + [[package]] 4407 4414 name = "serde-untagged" 4408 4415 version = "0.1.9" 4409 4416 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4596 4603 "roxmltree 0.21.1", 4597 4604 "rust-analyzer", 4598 4605 "serde", 4606 + "serde-aux", 4599 4607 "serde_cbor", 4600 4608 "serde_json", 4601 4609 "slug", 4602 4610 "strum", 4603 4611 "strum_macros", 4604 - "symphonia", 4605 - "symphonia-bundle-flac", 4606 4612 "tiny-skia", 4607 4613 "tokio", 4608 4614 "toml 0.9.8", ··· 4840 4846 dependencies = [ 4841 4847 "kurbo", 4842 4848 "siphasher", 4843 - ] 4844 - 4845 - [[package]] 4846 - name = "symphonia" 4847 - version = "0.5.5" 4848 - source = "registry+https://github.com/rust-lang/crates.io-index" 4849 - checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" 4850 - dependencies = [ 4851 - "lazy_static", 4852 - "symphonia-bundle-flac", 4853 - "symphonia-codec-adpcm", 4854 - "symphonia-codec-pcm", 4855 - "symphonia-codec-vorbis", 4856 - "symphonia-core", 4857 - "symphonia-format-mkv", 4858 - "symphonia-format-ogg", 4859 - "symphonia-format-riff", 4860 - "symphonia-metadata", 4861 - ] 4862 - 4863 - [[package]] 4864 - name = "symphonia-bundle-flac" 4865 - version = "0.5.5" 4866 - source = "registry+https://github.com/rust-lang/crates.io-index" 4867 - checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976" 4868 - dependencies = [ 4869 - "log", 4870 - "symphonia-core", 4871 - "symphonia-metadata", 4872 - "symphonia-utils-xiph", 4873 - ] 4874 - 4875 - [[package]] 4876 - name = "symphonia-codec-adpcm" 4877 - version = "0.5.5" 4878 - source = "registry+https://github.com/rust-lang/crates.io-index" 4879 - checksum = "2dddc50e2bbea4cfe027441eece77c46b9f319748605ab8f3443350129ddd07f" 4880 - dependencies = [ 4881 - "log", 4882 - "symphonia-core", 4883 - ] 4884 - 4885 - [[package]] 4886 - name = "symphonia-codec-pcm" 4887 - version = "0.5.5" 4888 - source = "registry+https://github.com/rust-lang/crates.io-index" 4889 - checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95" 4890 - dependencies = [ 4891 - "log", 4892 - "symphonia-core", 4893 - ] 4894 - 4895 - [[package]] 4896 - name = "symphonia-codec-vorbis" 4897 - version = "0.5.5" 4898 - source = "registry+https://github.com/rust-lang/crates.io-index" 4899 - checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73" 4900 - dependencies = [ 4901 - "log", 4902 - "symphonia-core", 4903 - "symphonia-utils-xiph", 4904 - ] 4905 - 4906 - [[package]] 4907 - name = "symphonia-core" 4908 - version = "0.5.5" 4909 - source = "registry+https://github.com/rust-lang/crates.io-index" 4910 - checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af" 4911 - dependencies = [ 4912 - "arrayvec", 4913 - "bitflags 1.3.2", 4914 - "bytemuck", 4915 - "lazy_static", 4916 - "log", 4917 - ] 4918 - 4919 - [[package]] 4920 - name = "symphonia-format-mkv" 4921 - version = "0.5.5" 4922 - source = "registry+https://github.com/rust-lang/crates.io-index" 4923 - checksum = "122d786d2c43a49beb6f397551b4a050d8229eaa54c7ddf9ee4b98899b8742d0" 4924 - dependencies = [ 4925 - "lazy_static", 4926 - "log", 4927 - "symphonia-core", 4928 - "symphonia-metadata", 4929 - "symphonia-utils-xiph", 4930 - ] 4931 - 4932 - [[package]] 4933 - name = "symphonia-format-ogg" 4934 - version = "0.5.5" 4935 - source = "registry+https://github.com/rust-lang/crates.io-index" 4936 - checksum = "2b4955c67c1ed3aa8ae8428d04ca8397fbef6a19b2b051e73b5da8b1435639cb" 4937 - dependencies = [ 4938 - "log", 4939 - "symphonia-core", 4940 - "symphonia-metadata", 4941 - "symphonia-utils-xiph", 4942 - ] 4943 - 4944 - [[package]] 4945 - name = "symphonia-format-riff" 4946 - version = "0.5.5" 4947 - source = "registry+https://github.com/rust-lang/crates.io-index" 4948 - checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f" 4949 - dependencies = [ 4950 - "extended", 4951 - "log", 4952 - "symphonia-core", 4953 - "symphonia-metadata", 4954 - ] 4955 - 4956 - [[package]] 4957 - name = "symphonia-metadata" 4958 - version = "0.5.5" 4959 - source = "registry+https://github.com/rust-lang/crates.io-index" 4960 - checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16" 4961 - dependencies = [ 4962 - "encoding_rs", 4963 - "lazy_static", 4964 - "log", 4965 - "symphonia-core", 4966 - ] 4967 - 4968 - [[package]] 4969 - name = "symphonia-utils-xiph" 4970 - version = "0.5.5" 4971 - source = "registry+https://github.com/rust-lang/crates.io-index" 4972 - checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16" 4973 - dependencies = [ 4974 - "symphonia-core", 4975 - "symphonia-metadata", 4976 4849 ] 4977 4850 4978 4851 [[package]]

+1 -2

Cargo.toml

··· 108 108 axum = { version = "0.8.6", optional = true, features = ["json"] } 109 109 quick-xml = "0.38.3" 110 110 vgv = { git = "https://github.com/gwennlbh/vgvf", version = "0.1.0", optional = true} 111 - symphonia-bundle-flac = { version = "0.5.5" } 112 - symphonia = { version = "0.5.5" } 111 + serde-aux = "4.7.0" 113 112 114 113 115 114 [dev-dependencies]

+1 -1

Justfile

··· 27 27 cp shapemaker {{install_at}} 28 28 29 29 example-video out="out.mp4" args='': 30 - RUST_BACKTRACE=full ./shapemaker test-video --colors examples/colorschemes/palenight.css {{out}} --sync-with examples/schedule-hell/schedule-hell.midi --audio examples/schedule-hell/schedule-hell.flac --grid-size 16x10 --resolution 480 {{args}} 30 + RUST_BACKTRACE=full ./shapemaker test-video --colors examples/colorschemes/palenight.css {{out}} --sync-with examples/schedule-hell/schedule-hell.midi --audio examples/schedule-hell/schedule-hell.wav --grid-size 16x10 --resolution 480 {{args}} 31 31 32 32 [working-directory: 'paper'] 33 33 paper:

+1 -1

examples/schedule-hell-backbone/src/main.rs

··· 25 25 canvas.object_sizes.dot_radius = 7.5; 26 26 27 27 let mut video = Video::<Ctx>::new(canvas); 28 - video.audiofile = "../schedule-hell/schedule-hell.flac".into(); 28 + video.audiofile = "../schedule-hell/schedule-hell.wav".into(); 29 29 video.fps = 60; 30 30 video.resolution = 480; 31 31 video.duration_override = Some(30_000);

+1

examples/schedule-hell/Cargo.toml

··· 8 8 itertools = "0.14.0" 9 9 pico-args = { version = "0.5.0", features = ["combined-flags", "eq-separator"] } 10 10 rand = "0.9.0" 11 + serde-aux = "4.7.0" 11 12 shapemaker = { path = "../..", features = ["video"] } 12 13 tokio = "1.48.0"

examples/schedule-hell/schedule-hell.flac

This is a binary file and will not be displayed.

examples/schedule-hell/schedule-hell.wav

This is a binary file and will not be displayed.

+2 -1

examples/schedule-hell/src/main.rs

··· 62 62 video.resolution = args.value_from_str("--resolution").ok().unwrap_or(480); 63 63 video.fps = args.value_from_str("--fps").ok().unwrap_or(30); 64 64 65 - video.audiofile = PathBuf::from("schedule-hell.flac"); 65 + video.audiofile = PathBuf::from("schedule-hell.wav"); 66 66 video = video 67 67 .sync_audio_with("schedule-hell.midi") 68 + .sync_audio_with("schedule-hell.wav") 68 69 .with_init_scene(scenes::intro()) 69 70 .with_marked_scene(scenes::first_break()) 70 71 .when_remaining(10, &|canvas, _| {

+6 -1

src/synchronization/audio.rs

··· 76 76 77 77 impl Display for SyncData { 78 78 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 79 - write!(f, "SyncData @ {} bpm, {} stems", self.bpm, self.stems.len()) 79 + write!( 80 + f, 81 + "SyncData @ {} bpm, {} stems", 82 + self.bpm.map_or("?".to_string(), |bpm| bpm.to_string()), 83 + self.stems.len() 84 + ) 80 85 } 81 86 } 82 87

+62 -27

src/synchronization/cue_markers.rs

··· 1 1 use crate::synchronization::sync::Syncable; 2 - use std::{collections::HashMap, fs::File, path::PathBuf}; 3 - use symphonia::core::{formats::FormatReader, io::MediaSourceStream}; 4 - use symphonia::default::formats::{FlacReader, WavReader}; 2 + use serde::Deserialize; 3 + use serde_aux::field_attributes::deserialize_number_from_string; 4 + use std::{ 5 + collections::HashMap, fs::File, io::Read, path::PathBuf, process::Stdio, 6 + }; 5 7 6 8 use super::sync::TimestampMS; 7 9 ··· 9 11 pub path: PathBuf, 10 12 } 11 13 14 + #[derive(Debug, Deserialize)] 15 + struct FFprobeChapterTags { 16 + title: String, 17 + } 18 + 19 + #[derive(Debug, Deserialize)] 20 + struct FFprobeChapter { 21 + // id: usize, 22 + // time_base: String, 23 + 24 + // start: usize, 25 + #[serde(deserialize_with = "deserialize_number_from_string")] 26 + start_time: f32, 27 + 28 + // end: usize, 29 + // #[serde(deserialize_with = "deserialize_number_from_string")] 30 + // end_time: f32, 31 + 32 + tags: FFprobeChapterTags, 33 + } 34 + 35 + #[derive(Debug, Deserialize)] 36 + struct FFprobeOutput { 37 + chapters: Vec<FFprobeChapter>, 38 + } 39 + 12 40 impl Syncable for CueMarkersSynchronizer { 13 41 fn new(path: impl Into<PathBuf>) -> Self { 14 42 Self { path: path.into() } ··· 18 46 &self, 19 47 progress: Option<&indicatif::ProgressBar>, 20 48 ) -> super::sync::SyncData { 21 - let markers: HashMap<TimestampMS, String> = HashMap::new(); 49 + let mut ffprobe = std::process::Command::new("ffprobe") 50 + .args(["-v", "error"]) 51 + .args(["-i", &self.path.to_string_lossy()]) 52 + .args(["-output_format", "json"]) 53 + .arg("-show_chapters") 54 + .stdout(Stdio::piped()) 55 + .spawn() 56 + .expect(&format!( 57 + "Couldn't run ffprobe to get chapters of {:?}", 58 + self.path 59 + )); 22 60 23 - let file = File::open(&self.path) 24 - .expect(&format!("Failed to open {:?} for CUE analysis", self.path)); 25 - let stream = MediaSourceStream::new(Box::new(file), Default::default()); 26 - let reader: Box<dyn FormatReader> = 27 - match self.path.extension().and_then(|s| s.to_str()) { 28 - Some("wav") => Box::new( 29 - WavReader::try_new(stream, &Default::default()) 30 - .expect("Failed to create WAV reader for CUE analysis"), 31 - ), 32 - Some("flac") => Box::new( 33 - FlacReader::try_new(stream, &Default::default()) 34 - .expect("Failed to create FLAC reader for CUE analysis"), 35 - ), 36 - _ => panic!("Unsupported audio format for CUE analysis"), 37 - }; 61 + let mut raw_output = String::new(); 62 + ffprobe 63 + .stdout 64 + .take() 65 + .expect("Coudln't get stdout of ffprobe run") 66 + .read_to_string(&mut raw_output) 67 + .expect("Couldn't read ffprobe stdout"); 38 68 39 - for cue in reader.cues() { 40 - panic!("Found cue {cue:?}"); 41 - if let Some(pb) = progress { 42 - pb.set_message(format!("{cue:?}")); 43 - } 44 - } 69 + let output: FFprobeOutput = 70 + serde_json::from_str(&raw_output).expect("Invalid ffprobe output"); 45 71 46 72 super::sync::SyncData { 47 73 stems: HashMap::new(), 48 - markers, 49 - bpm: 120, 74 + bpm: None, 75 + markers: output 76 + .chapters 77 + .iter() 78 + .map(|ch| { 79 + ( 80 + (ch.start_time.to_owned() * 1_000.0) as TimestampMS, 81 + ch.tags.title.clone(), 82 + ) 83 + }) 84 + .collect(), 50 85 } 51 86 } 52 87 }

+1 -3

src/synchronization/midi.rs

··· 34 34 let (now, notes_per_instrument, markers) = 35 35 load_midi_file(&self.midi_path, progressbar); 36 36 37 - println!("Found markers {markers:?}"); 38 - 39 37 SyncData { 40 38 markers, 41 - bpm: tempo_to_bpm(now.tempo), 39 + bpm: Some(tempo_to_bpm(now.tempo)), 42 40 stems: HashMap::from_iter(notes_per_instrument.iter().map( 43 41 |(name, notes)| { 44 42 let mut notes_per_ms =

+2 -2

src/synchronization/sync.rs

··· 13 13 pub struct SyncData { 14 14 pub stems: HashMap<String, Stem>, 15 15 pub markers: HashMap<TimestampMS, String>, 16 - pub bpm: usize, 16 + pub bpm: Option<usize>, 17 17 } 18 18 19 19 impl SyncData { 20 20 pub fn union(self, other: SyncData) -> Self { 21 21 let mut combined = Self::default(); 22 22 23 - combined.bpm = other.bpm; 23 + combined.bpm = other.bpm.or(self.bpm); 24 24 25 25 combined.stems.extend(self.stems); 26 26 combined.stems.extend(other.stems);

+19 -1

src/ui.rs

··· 1 1 use crate::video::engine::EngineProgression; 2 2 use console::Style; 3 3 use indicatif::{ProgressBar, ProgressStyle}; 4 + use itertools::Itertools; 4 5 use std::borrow::Cow; 6 + use std::collections::HashMap; 5 7 use std::sync::{Arc, Mutex}; 6 8 use std::thread::{self, JoinHandle}; 7 9 use std::time; 8 10 9 11 pub const PROGRESS_BARS_STYLE: &str = 10 - "{prefix:>12.bold.cyan} {percent:03}% [{bar:25}] {msg} ({elapsed} ago)"; 12 + "\x1b]9;4;1;{percent}\x1b\\{prefix:>12.bold.cyan} {percent:03}% [{bar:25}] {msg} ({elapsed} ago)"; 11 13 12 14 pub struct Spinner { 13 15 pub spinner: ProgressBar, ··· 60 62 .unwrap() 61 63 .progress_chars("=> "), 62 64 ) 65 + .with_finish(indicatif::ProgressFinish::WithMessage( 66 + "\x1b]9;4;0\x1b\\".into(), 67 + )) 63 68 } 64 69 65 70 pub trait Log { ··· 135 140 } 136 141 } 137 142 } 143 + 144 + pub fn display_counts(counts: HashMap<impl std::fmt::Display, usize>) -> String { 145 + counts 146 + .iter() 147 + .filter_map(|(name, &count)| { 148 + if count > 0 { 149 + Some(format!("{count} {name}")) 150 + } else { 151 + None 152 + } 153 + }) 154 + .join(", ") 155 + }

+4 -10

src/video/engine.rs

··· 44 44 beat_fractional: 0.0, 45 45 timestamp: "00:00:00.000".to_string(), 46 46 ms: 0, 47 - bpm: self.syncdata.bpm, 48 47 syncdata: &self.syncdata, 49 48 extra: AdditionalContext::default(), 50 49 later_hooks: vec![], 51 50 audiofile: self.audiofile.clone(), 52 51 duration_override: self.duration_override, 53 52 scene_started_at_ms: None, 53 + bpm: self 54 + .syncdata 55 + .bpm 56 + .expect("No sync source could determine the BPM"), 54 57 }; 55 58 56 59 let mut canvas = self.initial_canvas.clone(); ··· 92 95 93 96 if let EngineControl::RenderFromCanvas(new_canvas) = control { 94 97 canvas = new_canvas; 95 - } 96 - 97 - if context.marker() != "" { 98 - self.progress_bar.println(format!( 99 - "{}: marker {}", 100 - context.timestamp, 101 - context.marker() 102 - )); 103 98 } 104 99 105 100 if context.marker().starts_with(':') { ··· 178 173 179 174 output.send(EngineOutput::Finished)?; 180 175 181 - println!("Rendered {rendered_frames_count} frames"); 182 176 Ok(rendered_frames_count) 183 177 } 184 178

+14 -8

src/video/video.rs

··· 4 4 midi::MidiSynchronizer, 5 5 sync::{SyncData, Syncable}, 6 6 }, 7 - ui::{self, Log}, 7 + ui::{self, display_counts, Log}, 8 8 video::hooks::{AttachHooks, CommandAction, Hook}, 9 9 Canvas, Scene, 10 10 }; 11 11 use indicatif::ProgressBar; 12 12 use measure_time::debug_time; 13 - use std::{fmt::Formatter, path::PathBuf}; 13 + use std::{collections::HashMap, fmt::Formatter, path::PathBuf}; 14 14 15 15 pub struct Command<C> { 16 16 pub name: String, ··· 93 93 self.progress_bar.log( 94 94 "Loaded", 95 95 &format!( 96 - "{} notes from {file_path:?}", 97 - syncdata 98 - .stems 99 - .values() 100 - .map(|v| v.notes.len()) 101 - .sum::<usize>(), 96 + "{} from {file_path:?}", 97 + display_counts(HashMap::from([ 98 + ("markers", syncdata.markers.len()), 99 + ( 100 + "notes", 101 + syncdata 102 + .stems 103 + .values() 104 + .map(|v| v.notes.len()) 105 + .sum::<usize>() 106 + ), 107 + ])), 102 108 ), 103 109 ); 104 110

Configure Feed

Configure Feed