xil/cmd/
stem_verify.rs

1//! `xil stem-verify` — a JSON report of every stem's attributes and,
2//! optionally, its Faster-Whisper transcript. Port of
3//! `XILU015_stem_verify.py`.
4
5use std::ffi::OsString;
6use std::fs;
7use std::path::{Path, PathBuf};
8
9use anyhow::{anyhow, bail};
10use clap::Parser;
11use serde_json::{Map, Value};
12use xil_core::fsutil::basename;
13use xil_core::pyfmt::round_to;
14use xil_core::pyjson::{dumps, py_float, Style};
15use xil_core::workspace::{resolve_slug, resolve_venv_python, workspace_root};
16use xil_core::{banner, log};
17use xil_workers::Worker;
18
19#[derive(Parser)]
20#[command(
21    name = "xil-stem-verify",
22    about = "Scan a stems folder and produce a JSON report with file attributes and Whisper transcriptions."
23)]
24struct Args {
25    /// Show slug (default: resolved from project.json or XIL_PROJECTROOT)
26    #[arg(long, short = 's', value_name = "SLUG")]
27    show: Option<String>,
28    /// Episode tag, e.g. S01E01 (required unless --stems-dir is set)
29    #[arg(long, short = 'e', value_name = "TAG")]
30    episode: Option<String>,
31    /// Override stems directory (default: <workspace>/stems/<slug>/<episode>/)
32    #[arg(long, value_name = "DIR")]
33    stems_dir: Option<String>,
34    /// Output JSON path (default: <workspace>/parsed/<slug>/stem_verify_<episode>.json)
35    #[arg(long, short = 'o', value_name = "FILE")]
36    output: Option<String>,
37    /// Path to venv-whisper Python executable (auto-detected if omitted)
38    #[arg(long, value_name = "PATH")]
39    whisper_python: Option<String>,
40    /// Whisper model size: tiny|base|small|medium|large-v3|large-v3-turbo (default: large-v3-turbo)
41    #[arg(long, default_value = "large-v3-turbo", value_name = "SIZE")]
42    model: String,
43    /// Language hint for Whisper. Use 'auto' for automatic detection (default: en)
44    #[arg(long, default_value = "en", value_name = "LANG")]
45    language: String,
46    /// Whisper beam size (default: 5)
47    #[arg(
48        long,
49        default_value_t = 5,
50        value_name = "N",
51        allow_negative_numbers = true
52    )]
53    beam_size: i64,
54    /// Compute device for Whisper (default: cuda)
55    #[arg(long, default_value = "cuda", value_parser = ["cuda", "cpu"])]
56    device: String,
57    /// Skip Whisper transcription; output file attributes only
58    #[arg(long)]
59    no_transcribe: bool,
60}
61
62/// `int(s)` as Python parses a stem prefix.
63fn py_int(s: &str) -> Option<i64> {
64    let t = s.trim();
65    let (neg, digits) = match t.strip_prefix('-') {
66        Some(r) => (true, r),
67        None => (false, t.strip_prefix('+').unwrap_or(t)),
68    };
69    if digits.is_empty()
70        || digits.starts_with('_')
71        || digits.ends_with('_')
72        || digits.contains("__")
73    {
74        return None;
75    }
76    let clean: String = digits.chars().filter(|&c| c != '_').collect();
77    if !clean.chars().all(|c| c.is_ascii_digit()) {
78        return None;
79    }
80    clean.parse::<i64>().ok().map(|n| if neg { -n } else { n })
81}
82
83/// `_parse_stem_filename` → `(seq, scene, speaker)`.
84fn parse_stem_filename(filename: &str) -> (Option<i64>, Option<String>, Option<String>) {
85    let name = if filename.to_lowercase().ends_with(".mp3") {
86        let cut: Vec<char> = filename.chars().collect();
87        cut[..cut.len() - 4].iter().collect::<String>()
88    } else {
89        filename.to_string()
90    };
91    let parts: Vec<&str> = name.splitn(3, '_').collect();
92    if parts.len() == 3 {
93        if let Some(seq) = py_int(parts[0]) {
94            return (
95                Some(seq),
96                Some(parts[1].to_string()),
97                Some(parts[2].to_string()),
98            );
99        }
100    }
101    (py_int(parts[0]), None, None)
102}
103
104/// `_mp3_metadata` → `(round(length, 3), bitrate // 1000)`, or both `None`.
105fn mp3_metadata(path: &Path) -> (Option<f64>, Option<u64>) {
106    match xil_audio::mpeg::info(path) {
107        Ok(info) => (Some(round_to(info.length, 3)), Some(info.bitrate / 1000)),
108        Err(_) => (None, None),
109    }
110}
111
112/// `_WhisperClient` — strict about the first line being the ready message.
113struct Whisper {
114    worker: Worker,
115}
116
117impl Whisper {
118    fn start(python: &str, script: &Path, device: &str, model: &str) -> anyhow::Result<Whisper> {
119        let mut worker = Worker::spawn(
120            Path::new(python),
121            script,
122            &[device.to_string(), model.to_string()],
123        )?;
124        let line = worker.read_line()?;
125        let startup: Value = serde_json::from_str(&line)
126            .map_err(|e| anyhow!("json.decoder.JSONDecodeError: {e}"))?;
127        let ready = xil_workers::is_truthy(startup.get("ready"));
128        if !ready {
129            bail!(
130                "Whisper worker failed to start: {}",
131                dumps(&startup, Style::COMPACT)
132            );
133        }
134        let field = |k: &str| match startup.get(k) {
135            Some(Value::String(s)) => s.clone(),
136            Some(Value::Null) | None => "None".into(),
137            Some(v) => dumps(v, Style::COMPACT),
138        };
139        log::info(&format!(
140            "Whisper worker ready — model={} device={}",
141            field("model"),
142            field("device")
143        ));
144        Ok(Whisper { worker })
145    }
146
147    fn transcribe(
148        &mut self,
149        audio_path: &str,
150        language: Option<&str>,
151        beam_size: i64,
152    ) -> anyhow::Result<Value> {
153        let mut req = Map::new();
154        req.insert("audio_path".into(), audio_path.into());
155        req.insert(
156            "language".into(),
157            language.map(Value::from).unwrap_or(Value::Null),
158        );
159        req.insert("beam_size".into(), beam_size.into());
160        self.worker.send(&Value::Object(req))?;
161        let resp: Value = serde_json::from_str(&self.worker.read_line()?)?;
162        if let Some(err) = resp.get("error") {
163            let msg = match err {
164                Value::String(s) => s.clone(),
165                other => dumps(other, Style::COMPACT),
166            };
167            bail!(
168                "Whisper error on {}: {msg}",
169                basename(Path::new(audio_path))
170            );
171        }
172        Ok(resp)
173    }
174}
175
176fn process_files(
177    files: &[PathBuf],
178    mut whisper: Option<&mut Whisper>,
179    language: Option<&str>,
180    beam_size: i64,
181) -> anyhow::Result<Vec<Value>> {
182    let total = files.len();
183    let mut records = Vec::new();
184    for (i, mp3) in files.iter().enumerate() {
185        let name = basename(mp3);
186        log::info(&format!("[{}/{total}] {name}", i + 1));
187        let (seq, scene, speaker) = parse_stem_filename(&name);
188        if seq.is_none() {
189            log::warning(&format!(
190                "Unexpected filename format (skipping seq/scene/speaker parse): {name}"
191            ));
192        }
193        let (duration, bitrate) = mp3_metadata(mp3);
194        let digest = crate::cmd::mp3_hash::hash_file(mp3)?;
195        let mut transcript = Value::Null;
196        if let Some(w) = whisper.as_deref_mut() {
197            match w.transcribe(&mp3.to_string_lossy(), language, beam_size) {
198                Ok(resp) => {
199                    let mut t = Map::new();
200                    for k in ["text", "language", "language_probability", "segments"] {
201                        match resp.get(k) {
202                            Some(v) => {
203                                t.insert(k.into(), v.clone());
204                            }
205                            None => {
206                                transcript = Value::Null;
207                                log::warning(&format!("Transcription failed for {name}: '{k}'"));
208                                t.clear();
209                                break;
210                            }
211                        }
212                    }
213                    if !t.is_empty() {
214                        transcript = Value::Object(t);
215                    }
216                }
217                Err(e) => log::warning(&format!("Transcription failed for {name}: {e}")),
218            }
219        }
220        let mut r = Map::new();
221        r.insert("filename".into(), name.clone().into());
222        r.insert(
223            "path".into(),
224            fs::canonicalize(mp3)?.to_string_lossy().into_owned().into(),
225        );
226        r.insert("seq".into(), seq.map(Value::from).unwrap_or(Value::Null));
227        r.insert(
228            "scene".into(),
229            scene.map(Value::from).unwrap_or(Value::Null),
230        );
231        r.insert(
232            "speaker".into(),
233            speaker.map(Value::from).unwrap_or(Value::Null),
234        );
235        r.insert("size_bytes".into(), fs::metadata(mp3)?.len().into());
236        r.insert(
237            "duration_seconds".into(),
238            duration.map(py_float).unwrap_or(Value::Null),
239        );
240        r.insert(
241            "bitrate_kbps".into(),
242            bitrate.map(Value::from).unwrap_or(Value::Null),
243        );
244        r.insert("sha256".into(), digest.into());
245        r.insert("transcript".into(), transcript);
246        records.push(Value::Object(r));
247    }
248    Ok(records)
249}
250
251pub fn run(args: &[OsString]) -> anyhow::Result<i32> {
252    log::init("stem-verify");
253    let a: Args = match super::parse_or_exit("xil-stem-verify", args) {
254        Ok(a) => a,
255        Err(code) => return Ok(code),
256    };
257    let _banner = banner::begin(super::prog(), &super::argv_line(args));
258    execute(&a)
259}
260
261fn execute(a: &Args) -> anyhow::Result<i32> {
262    if a.episode.is_none() && a.stems_dir.is_none() {
263        log::error("--episode or --stems-dir is required");
264        return Ok(1);
265    }
266    let workspace = workspace_root();
267    let slug = resolve_slug(a.show.as_deref(), "project.json");
268    let stems_dir = match a.stems_dir.as_ref().filter(|s| !s.is_empty()) {
269        Some(d) => PathBuf::from(d),
270        None => workspace
271            .join("stems")
272            .join(&slug)
273            .join(a.episode.clone().unwrap_or_default()),
274    };
275    if !stems_dir.is_dir() {
276        log::error(&format!(
277            "Stems directory not found: {}",
278            stems_dir.display()
279        ));
280        return Ok(1);
281    }
282    let episode = a.episode.clone().filter(|e| !e.is_empty());
283    let output_path = match (a.output.as_ref().filter(|s| !s.is_empty()), &episode) {
284        (Some(o), _) => PathBuf::from(o),
285        (None, Some(ep)) => workspace
286            .join("parsed")
287            .join(&slug)
288            .join(format!("stem_verify_{ep}.json")),
289        (None, None) => stems_dir.join("stem_verify_report.json"),
290    };
291    if let Some(parent) = output_path.parent().filter(|p| !p.as_os_str().is_empty()) {
292        fs::create_dir_all(parent)?;
293    }
294
295    let mut files: Vec<PathBuf> = fs::read_dir(&stems_dir)?
296        .filter_map(Result::ok)
297        .map(|e| e.path())
298        .filter(|p| {
299            let n = basename(p);
300            matches!(n.rfind('.'), Some(i) if i > 0 && i < n.len() - 1 && n[i..].to_lowercase() == ".mp3")
301        })
302        .collect();
303    xil_core::fsutil::sort_py(&mut files);
304    if files.is_empty() {
305        log::error(&format!("No MP3 files found in {}", stems_dir.display()));
306        return Ok(1);
307    }
308    log::info(&format!(
309        "Found {} MP3 files in {}",
310        files.len(),
311        stems_dir.display()
312    ));
313
314    let transcribe = !a.no_transcribe;
315    let language = if a.language == "auto" {
316        None
317    } else {
318        Some(a.language.as_str())
319    };
320    let records = if transcribe {
321        let package_dir = crate::workers::python_package_dir();
322        let Some(python) = resolve_venv_python(
323            "venv-whisper",
324            a.whisper_python.as_deref(),
325            package_dir.as_deref(),
326        ) else {
327            log::error(
328                "Cannot find venv-whisper Python. Pass --whisper-python PATH, set XIL_CODEROOT to the directory containing venv-whisper/, or build it with: xil setup whisper. Use --no-transcribe to skip transcription.",
329            );
330            return Ok(1);
331        };
332        let script = package_dir
333            .map(|d| xil_workers::worker_script(&d, "whisper_worker.py"))
334            .ok_or_else(|| {
335                anyhow!("cannot locate the xil_pipeline package that ships whisper_worker.py")
336            })?;
337        let mut whisper = Whisper::start(&python, &script, &a.device, &a.model)?;
338        let records = process_files(&files, Some(&mut whisper), language, a.beam_size);
339        whisper.worker.close()?;
340        records?
341    } else {
342        process_files(&files, None, language, a.beam_size)?
343    };
344
345    let total_duration: f64 = records
346        .iter()
347        .map(|r| r["duration_seconds"].as_f64().unwrap_or(0.0))
348        .sum();
349    let mut report = Map::new();
350    report.insert("show".into(), slug.into());
351    report.insert(
352        "episode".into(),
353        episode.unwrap_or_else(|| basename(&stems_dir)).into(),
354    );
355    report.insert(
356        "generated".into(),
357        chrono::Local::now()
358            .format("%Y-%m-%dT%H:%M:%S")
359            .to_string()
360            .into(),
361    );
362    report.insert(
363        "stems_dir".into(),
364        fs::canonicalize(&stems_dir)?
365            .to_string_lossy()
366            .into_owned()
367            .into(),
368    );
369    report.insert(
370        "whisper_model".into(),
371        if transcribe {
372            a.model.clone().into()
373        } else {
374            Value::Null
375        },
376    );
377    report.insert("file_count".into(), files.len().into());
378    report.insert(
379        "total_duration_seconds".into(),
380        py_float(round_to(total_duration, 3)),
381    );
382    report.insert("files".into(), Value::Array(records));
383    fs::write(&output_path, dumps(&Value::Object(report), Style::INDENT2))?;
384
385    log::info(&format!("Written: {}", output_path.display()));
386    log::info(&format!(
387        "Total stems: {}  Total duration: {total_duration:.1}s",
388        files.len()
389    ));
390    Ok(0)
391}
392
393/// The clap definition behind `--help`, for man pages.
394pub fn command() -> clap::Command {
395    <Args as clap::CommandFactory>::command()
396}
397
398#[cfg(test)]
399mod tests {
400    use super::*;
401
402    #[test]
403    fn stem_names_parse_like_python() {
404        assert_eq!(
405            parse_stem_filename("003_cold-open_adam.mp3"),
406            (Some(3), Some("cold-open".into()), Some("adam".into()))
407        );
408        assert_eq!(
409            parse_stem_filename("001_Chapter 1.mp3"),
410            (Some(1), None, None)
411        );
412        assert_eq!(parse_stem_filename("intro.MP3"), (None, None, None));
413        assert_eq!(
414            parse_stem_filename("010_act1_scene_host.mp3"),
415            (Some(10), Some("act1".into()), Some("scene_host".into()))
416        );
417    }
418}
xil/cmd/stem_verify.rs

xil/cmd/
stem_verify.rs
