xil/cmd/
stem_compare.rs

1//! `xil stem-compare` — flag dialogue stems whose Whisper transcript
2//! strays from the scripted line. Port of `XILU016_stem_compare.py`.
3
4use std::ffi::OsString;
5use std::fs;
6use std::path::{Path, PathBuf};
7
8use clap::Parser;
9use regex::Regex;
10use serde_json::{Map, Value};
11use xil_core::pycsv;
12use xil_core::pyfmt::{head, round_to};
13use xil_core::pyjson::{dumps, py_float, Style};
14use xil_core::textsim::SequenceMatcher;
15use xil_core::workspace::{resolve_slug, workspace_root};
16use xil_core::{banner, log};
17
18#[derive(Parser)]
19#[command(
20    name = "xil-stem-compare",
21    about = "Cross-reference Whisper transcripts against parsed script dialogue to flag garbled stems."
22)]
23struct Args {
24    /// Show slug (default: resolved from project.json)
25    #[arg(long, short = 's', value_name = "SLUG")]
26    show: Option<String>,
27    /// Episode tag, e.g. S01E01 (derives both JSON paths if not overridden)
28    #[arg(long, short = 'e', value_name = "TAG")]
29    episode: Option<String>,
30    /// Path to stem_verify JSON (default: <workspace>/parsed/<slug>/stem_verify_<episode>.json)
31    #[arg(long, value_name = "FILE")]
32    stem_verify: Option<String>,
33    /// Path to parsed script JSON (default: <workspace>/parsed/<slug>/parsed_<episode>.json)
34    #[arg(long, value_name = "FILE")]
35    parsed: Option<String>,
36    /// Similarity below this marks a stem as garbled (default: 0.75)
37    #[arg(
38        long,
39        default_value_t = 0.75,
40        value_name = "FLOAT",
41        allow_negative_numbers = true
42    )]
43    threshold: f64,
44    /// Write full JSON report to this file (in addition to terminal output)
45    #[arg(long, short = 'o', value_name = "FILE")]
46    output: Option<String>,
47    /// Print flagged entries as CSV to stdout instead of the banner summary
48    #[arg(long)]
49    csv: bool,
50}
51
52/// `_normalize`: lower-case, punctuation to spaces, runs of space collapsed.
53fn normalize(text: &str) -> String {
54    static PUNCT: std::sync::OnceLock<Regex> = std::sync::OnceLock::new();
55    static SPACE: std::sync::OnceLock<Regex> = std::sync::OnceLock::new();
56    let lower = text.to_lowercase();
57    let t = PUNCT
58        .get_or_init(|| Regex::new(r"[^\w\s]").unwrap())
59        .replace_all(&lower, " ");
60    SPACE
61        .get_or_init(|| Regex::new(r"\s+").unwrap())
62        .replace_all(&t, " ")
63        .trim()
64        .to_string()
65}
66
67fn similarity(a: &str, b: &str) -> f64 {
68    SequenceMatcher::new(&normalize(a), &normalize(b)).ratio()
69}
70
71struct Flag {
72    seq: Value,
73    section: Value,
74    scene: Value,
75    speaker: Value,
76    status: &'static str,
77    similarity: Option<f64>,
78    original: Value,
79    transcript: Value,
80}
81
82impl Flag {
83    fn json(&self) -> Value {
84        let mut m = Map::new();
85        m.insert("seq".into(), self.seq.clone());
86        m.insert("section".into(), self.section.clone());
87        m.insert("scene".into(), self.scene.clone());
88        m.insert("speaker".into(), self.speaker.clone());
89        m.insert("status".into(), self.status.into());
90        m.insert(
91            "similarity".into(),
92            self.similarity.map(py_float).unwrap_or(Value::Null),
93        );
94        m.insert("original".into(), self.original.clone());
95        m.insert("transcript".into(), self.transcript.clone());
96        Value::Object(m)
97    }
98}
99
100const STATUSES: [&str; 5] = ["ok", "garbled", "silent", "no_stem", "not_transcribed"];
101
102fn load_json(p: &Path) -> anyhow::Result<Value> {
103    Ok(serde_json::from_str(&fs::read_to_string(p)?)?)
104}
105
106fn compare(
107    dialogue: &[Map<String, Value>],
108    stems: &indexmap::IndexMap<i64, Map<String, Value>>,
109    threshold: f64,
110) -> (Vec<Flag>, [usize; 5]) {
111    let mut counts = [0usize; 5];
112    let mut flags = Vec::new();
113    for entry in dialogue {
114        let seq_v = entry.get("seq").cloned().unwrap_or(Value::Null);
115        let get = |k: &str| entry.get(k).cloned().unwrap_or(Value::Null);
116        let original = get("text");
117        let make = |status, sim, transcript| Flag {
118            seq: seq_v.clone(),
119            section: get("section"),
120            scene: get("scene"),
121            speaker: get("speaker"),
122            status,
123            similarity: sim,
124            original: original.clone(),
125            transcript,
126        };
127        let Some(stem) = seq_v.as_i64().and_then(|s| stems.get(&s)) else {
128            counts[3] += 1;
129            flags.push(make("no_stem", None, Value::Null));
130            continue;
131        };
132        let transcript = match stem.get("transcript") {
133            None | Some(Value::Null) => {
134                counts[4] += 1;
135                flags.push(make("not_transcribed", None, Value::Null));
136                continue;
137            }
138            Some(t) => t,
139        };
140        let text = transcript.get("text").and_then(Value::as_str).unwrap_or("");
141        if text.trim().is_empty() {
142            counts[2] += 1;
143            flags.push(make("silent", Some(0.0), Value::String(String::new())));
144            continue;
145        }
146        let sim = round_to(similarity(original.as_str().unwrap_or(""), text), 4);
147        if sim < threshold {
148            counts[1] += 1;
149            flags.push(make("garbled", Some(sim), Value::String(text.to_string())));
150        } else {
151            counts[0] += 1;
152        }
153    }
154    (flags, counts)
155}
156
157fn str_of(v: &Value) -> String {
158    match v {
159        Value::String(s) => s.clone(),
160        Value::Null => "None".into(),
161        other => pycsv::cell(other),
162    }
163}
164
165fn print_summary(counts: &[usize; 5], flags: &[Flag], threshold: f64) {
166    let total: usize = counts.iter().sum();
167    let ok_pct = if total > 0 {
168        counts[0] as f64 / total as f64 * 100.0
169    } else {
170        0.0
171    };
172    log::info(&format!("  Threshold      : {threshold:.2}"));
173    log::info(&format!("  Dialogue stems : {total}"));
174    log::info(&format!("  OK             : {}  ({ok_pct:.1}%)", counts[0]));
175    log::info(&format!("  Garbled        : {}", counts[1]));
176    log::info(&format!("  Silent         : {}", counts[2]));
177    log::info(&format!("  No stem        : {}", counts[3]));
178    log::info(&format!("  Not transcribed: {}", counts[4]));
179    if flags.is_empty() {
180        log::info("");
181        log::info("  No issues found.");
182        return;
183    }
184    log::info("");
185    log::info("--- Flagged entries ---");
186    for f in flags {
187        let seq = match &f.seq {
188            Value::Number(n) if n.as_i64().is_some() => format!("{:03}", n.as_i64().unwrap()),
189            other => str_of(other),
190        };
191        let speaker_full = match &f.speaker {
192            Value::String(s) if !s.is_empty() => s.clone(),
193            _ => "?".into(),
194        };
195        let speaker = head(&speaker_full, 12);
196        let original = str_of(&f.original);
197        match f.status {
198            "garbled" => {
199                log::info(&format!(
200                    "[garbled] seq={seq}  {speaker:<12} sim={:.2}",
201                    f.similarity.unwrap_or(0.0)
202                ));
203                log::info(&format!("  ORIGINAL  : {original}"));
204                log::info(&format!("  TRANSCRIPT: {}", str_of(&f.transcript)));
205            }
206            "silent" => {
207                log::info(&format!("[silent]  seq={seq}  {speaker}"));
208                log::info(&format!("  ORIGINAL  : {original}"));
209            }
210            "no_stem" => {
211                log::info(&format!("[no_stem] seq={seq}  {speaker}"));
212                log::info(&format!("  ORIGINAL  : {original}"));
213            }
214            _ => {
215                log::info(&format!("[no_xscr] seq={seq}  {speaker}"));
216                log::info(&format!("  ORIGINAL  : {original}"));
217            }
218        }
219    }
220}
221
222fn print_csv(flags: &[Flag]) -> std::io::Result<()> {
223    use std::io::Write;
224    let mut out = std::io::stdout().lock();
225    let header: Vec<String> = [
226        "seq",
227        "section",
228        "scene",
229        "speaker",
230        "status",
231        "similarity",
232        "original",
233        "transcript",
234    ]
235    .iter()
236    .map(|s| s.to_string())
237    .collect();
238    pycsv::write_row(&mut out, &header)?;
239    for f in flags {
240        let sim = f.similarity.map(py_float).unwrap_or(Value::Null);
241        let row: Vec<String> = [
242            &f.seq,
243            &f.section,
244            &f.scene,
245            &f.speaker,
246            &Value::from(f.status),
247            &sim,
248            &f.original,
249            &f.transcript,
250        ]
251        .iter()
252        .map(|v| pycsv::cell(v))
253        .collect();
254        pycsv::write_row(&mut out, &row)?;
255    }
256    out.flush()
257}
258
259pub fn run(args: &[OsString]) -> anyhow::Result<i32> {
260    log::init("stem-compare");
261    let a: Args = match super::parse_or_exit("xil-stem-compare", args) {
262        Ok(a) => a,
263        Err(code) => return Ok(code),
264    };
265    let _banner = banner::begin(super::prog(), &super::argv_line(args));
266    execute(&a)
267}
268
269fn execute(a: &Args) -> anyhow::Result<i32> {
270    let workspace = workspace_root();
271    let slug = resolve_slug(a.show.as_deref(), "project.json");
272    let episode = a.episode.clone().filter(|e| !e.is_empty());
273    let episode_label = episode.clone().unwrap_or_else(|| "unknown".into());
274
275    let stem_verify_path = match a.stem_verify.as_ref().filter(|s| !s.is_empty()) {
276        Some(p) => PathBuf::from(p),
277        None => match &episode {
278            Some(ep) => workspace
279                .join("parsed")
280                .join(&slug)
281                .join(format!("stem_verify_{ep}.json")),
282            None => {
283                log::error("--episode is required unless --stem-verify is provided");
284                return Ok(1);
285            }
286        },
287    };
288    let parsed_path = match a.parsed.as_ref().filter(|s| !s.is_empty()) {
289        Some(p) => PathBuf::from(p),
290        None => match &episode {
291            Some(ep) => workspace
292                .join("parsed")
293                .join(&slug)
294                .join(format!("parsed_{ep}.json")),
295            None => {
296                log::error("--episode is required unless --parsed is provided");
297                return Ok(1);
298            }
299        },
300    };
301    if !stem_verify_path.exists() {
302        log::error(&format!(
303            "stem_verify JSON not found: {}",
304            stem_verify_path.display()
305        ));
306        return Ok(1);
307    }
308    if !parsed_path.exists() {
309        log::error(&format!("parsed JSON not found: {}", parsed_path.display()));
310        return Ok(1);
311    }
312    log::info(&format!("  stem_verify : {}", stem_verify_path.display()));
313    log::info(&format!("  parsed      : {}", parsed_path.display()));
314
315    let verify = load_json(&stem_verify_path)?;
316    let mut stems = indexmap::IndexMap::new();
317    for f in verify["files"].as_array().into_iter().flatten() {
318        let Some(o) = f.as_object() else { continue };
319        let Some(seq) = o.get("seq").filter(|s| !s.is_null()) else {
320            continue;
321        };
322        if o.get("speaker").and_then(Value::as_str) == Some("sfx") {
323            continue;
324        }
325        if let Some(s) = seq.as_i64() {
326            stems.insert(s, o.clone());
327        }
328    }
329    let parsed = load_json(&parsed_path)?;
330    let dialogue: Vec<Map<String, Value>> = parsed["entries"]
331        .as_array()
332        .into_iter()
333        .flatten()
334        .filter_map(|e| e.as_object())
335        .filter(|e| {
336            e.get("type").and_then(Value::as_str) == Some("dialogue")
337                && e.get("direction_type").map_or(true, Value::is_null)
338        })
339        .cloned()
340        .collect();
341
342    let (flags, counts) = compare(&dialogue, &stems, a.threshold);
343    if a.csv {
344        print_csv(&flags)?;
345    } else {
346        print_summary(&counts, &flags, a.threshold);
347    }
348
349    if let Some(out) = a.output.as_ref().filter(|s| !s.is_empty()) {
350        let out = PathBuf::from(out);
351        if let Some(parent) = out.parent().filter(|p| !p.as_os_str().is_empty()) {
352            fs::create_dir_all(parent)?;
353        }
354        let mut summary = Map::new();
355        summary.insert(
356            "total_dialogue".into(),
357            Value::from(counts.iter().sum::<usize>()),
358        );
359        for (k, n) in STATUSES.iter().zip(counts) {
360            summary.insert((*k).into(), Value::from(n));
361        }
362        let mut report = Map::new();
363        report.insert("show".into(), slug.clone().into());
364        report.insert("episode".into(), episode_label.into());
365        report.insert(
366            "generated".into(),
367            chrono::Local::now()
368                .format("%Y-%m-%dT%H:%M:%S")
369                .to_string()
370                .into(),
371        );
372        report.insert("threshold".into(), py_float(a.threshold));
373        report.insert(
374            "stem_verify_path".into(),
375            fs::canonicalize(&stem_verify_path)?
376                .to_string_lossy()
377                .into_owned()
378                .into(),
379        );
380        report.insert(
381            "parsed_path".into(),
382            fs::canonicalize(&parsed_path)?
383                .to_string_lossy()
384                .into_owned()
385                .into(),
386        );
387        report.insert("summary".into(), Value::Object(summary));
388        report.insert(
389            "flags".into(),
390            Value::Array(flags.iter().map(Flag::json).collect()),
391        );
392        fs::write(&out, dumps(&Value::Object(report), Style::INDENT2))?;
393        log::info(&format!("Written: {}", out.display()));
394    }
395    Ok(0)
396}
397
398/// The clap definition behind `--help`, for man pages.
399pub fn command() -> clap::Command {
400    <Args as clap::CommandFactory>::command()
401}
402
403#[cfg(test)]
404mod tests {
405    use super::*;
406
407    #[test]
408    fn normalize_strips_punctuation_and_case() {
409        assert_eq!(normalize("Hello, World!  It's  me."), "hello world it s me");
410        assert_eq!(similarity("Hello there!", "hello there"), 1.0);
411    }
412}
xil/cmd/stem_compare.rs

xil/cmd/
stem_compare.rs
