xil/cmd/
migrate.rs

1//! `xil migrate` — carry stems across a script revision. Port of
2//! `XILP007_stem_migrator.py`.
3
4use std::collections::{HashMap, HashSet};
5use std::ffi::OsString;
6use std::fs;
7use std::path::{Path, PathBuf};
8use std::sync::LazyLock;
9
10use clap::Parser;
11use regex::Regex;
12use serde_json::{Map, Value};
13use xil_core::workspace::{derive_paths, resolve_slug, workspace_root};
14use xil_core::{banner, log};
15
16static WHITESPACE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\s+").unwrap());
17static EM_DASH: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\s*\u{2014}\s*").unwrap());
18
19pub const COPY: &str = "COPY";
20pub const SPEAKER: &str = "SPEAKER";
21pub const NEW: &str = "NEW";
22pub const MISSING: &str = "MISSING";
23pub const SKIP: &str = "SKIP";
24
25/// Entry types that produce a stem file.
26const STEM_TYPES: [&str; 3] = ["dialogue", "direction", "silence"];
27
28/// Characters shown before a text snippet is truncated.
29const SNIP: usize = 55;
30
31const USAGE: &str =
32    "usage: xil-migrate [-h] [--episode TAG] [--tag TAG] [--show SHOW] [--old PATH]\n\
33                     \x20                  [--new PATH] [--stems DIR] [--orig-prefix ORIG_PREFIX]\n\
34                     \x20                  [--dry-run] [--strict] [--quiet]";
35
36fn parser_error(msg: &str) -> i32 {
37    eprintln!("{USAGE}");
38    eprintln!("xil-migrate: error: {msg}");
39    2
40}
41
42#[derive(Parser)]
43#[command(
44    name = "xil-migrate",
45    about = "Migrate episode stems from an old parsed JSON to a revised one. Copies unchanged stems to their \
46             new seq-numbered filenames; reports what still needs TTS/SFX generation. Run XILP002 afterwards \
47             to fill the gaps."
48)]
49struct Args {
50    /// Episode tag (e.g. S02E03); derives --old, --new, and --stems paths automatically
51    #[arg(long, value_name = "TAG")]
52    episode: Option<String>,
53    /// Raw tag for non-episodic content (e.g. V01C03, D01); same as --episode but skips format validation
54    #[arg(long, value_name = "TAG")]
55    tag: Option<String>,
56    /// Show name override (default: from project.json)
57    #[arg(long)]
58    show: Option<String>,
59    /// Old parsed JSON (overrides --episode)
60    #[arg(long, value_name = "PATH")]
61    old: Option<PathBuf>,
62    /// New parsed JSON (overrides --episode)
63    #[arg(long, value_name = "PATH")]
64    new: Option<PathBuf>,
65    /// Stems directory (overrides --episode)
66    #[arg(long, value_name = "DIR")]
67    stems: Option<PathBuf>,
68    /// Filename prefix for the old parsed JSON (default: orig_)
69    #[arg(long, default_value = "orig_")]
70    orig_prefix: String,
71    /// Show plan without copying any files
72    #[arg(long)]
73    dry_run: bool,
74    /// Exact text match only. Default is fuzzy: ignores em-dash/ellipsis variants so punctuation-only edits don't force unnecessary regen.
75    #[arg(long)]
76    strict: bool,
77    /// Print only the summary, not per-stem details
78    #[arg(long)]
79    quiet: bool,
80}
81
82type Entry = Map<String, Value>;
83
84/// What should happen to one new parsed entry.
85#[derive(Debug, Clone)]
86pub struct Action {
87    pub status: &'static str,
88    pub new_stem: String,
89    pub old_seq: Option<i64>,
90    pub old_stem: Option<String>,
91    pub reason: String,
92    pub new_text: String,
93}
94
95/// Collapse whitespace; in fuzzy mode also fold em-dash, ellipsis and
96/// curly quotes so punctuation-only edits do not force regeneration.
97pub fn normalize_text(text: Option<&str>, strict: bool) -> String {
98    let Some(text) = text else {
99        return String::new();
100    };
101    let text = WHITESPACE.replace_all(text.trim(), " ").into_owned();
102    if strict {
103        return text;
104    }
105    EM_DASH
106        .replace_all(&text, " - ")
107        .replace('\u{2026}', "...")
108        .replace(['\u{2018}', '\u{2019}'], "'")
109        .replace(['\u{201c}', '\u{201d}'], "\"")
110}
111
112/// Expected stem filename for a parsed entry. Preamble entries (seq < 0)
113/// keep the legacy `n`-prefixed form.
114pub fn make_stem_name(entry: &Entry) -> String {
115    let seq = entry.get("seq").and_then(Value::as_i64).unwrap_or(0);
116    let section = entry
117        .get("section")
118        .and_then(Value::as_str)
119        .filter(|s| !s.is_empty())
120        .unwrap_or("unknown");
121    let scene = entry
122        .get("scene")
123        .and_then(Value::as_str)
124        .filter(|s| !s.is_empty());
125    let speaker = entry
126        .get("speaker")
127        .and_then(Value::as_str)
128        .filter(|s| !s.is_empty())
129        .unwrap_or("sfx");
130
131    let prefix = if seq < 0 {
132        format!("n{:03}", seq.abs())
133    } else {
134        format!("{seq:03}")
135    };
136    let mid = match scene {
137        Some(s) => format!("{section}-{s}"),
138        None => section.to_string(),
139    };
140    if entry.get("type").and_then(Value::as_str) == Some("dialogue") {
141        format!("{prefix}_{mid}_{speaker}.mp3")
142    } else {
143        format!("{prefix}_{mid}_sfx.mp3")
144    }
145}
146
147fn snip(text: Option<&str>) -> String {
148    let Some(t) = text.filter(|t| !t.is_empty()) else {
149        return String::new();
150    };
151    let t = t.trim();
152    if t.chars().count() > SNIP {
153        format!("{}\u{2026}", t.chars().take(SNIP).collect::<String>())
154    } else {
155        t.to_string()
156    }
157}
158
159fn match_key(entry: &Entry, strict: bool) -> (String, String) {
160    let role = entry
161        .get("speaker")
162        .and_then(Value::as_str)
163        .filter(|s| !s.is_empty())
164        .unwrap_or("sfx");
165    (
166        normalize_text(entry.get("text").and_then(Value::as_str), strict),
167        role.to_string(),
168    )
169}
170
171struct Record {
172    entry: Entry,
173    old_stem: String,
174    exists: bool,
175}
176
177/// Compare old and new entries and produce one action per new entry.
178///
179/// Matching is two-phase: exact on (text, speaker), then a text-only
180/// fallback for dialogue so a speaker reassignment is reported as such
181/// rather than as a brand-new line.
182pub fn plan_migration(
183    old_entries: &[Entry],
184    new_entries: &[Entry],
185    stems_dir: &Path,
186    strict: bool,
187) -> Vec<Action> {
188    let mut exact: HashMap<(String, String), Record> = HashMap::new();
189    let mut text_only: HashMap<String, Record> = HashMap::new();
190    for entry in old_entries {
191        let etype = entry.get("type").and_then(Value::as_str).unwrap_or("");
192        if !STEM_TYPES.contains(&etype) {
193            continue;
194        }
195        let stem_name = make_stem_name(entry);
196        let exists = stems_dir.join(&stem_name).is_file();
197        let ekey = match_key(entry, strict);
198        // The first occurrence wins, so repeated cues favour reuse.
199        exact.entry(ekey).or_insert(Record {
200            entry: entry.clone(),
201            old_stem: stem_name.clone(),
202            exists,
203        });
204        let tkey = normalize_text(entry.get("text").and_then(Value::as_str), strict);
205        text_only.entry(tkey).or_insert(Record {
206            entry: entry.clone(),
207            old_stem: stem_name,
208            exists,
209        });
210    }
211
212    let mut used_exact: HashSet<(String, String)> = HashSet::new();
213    let mut used_text: HashSet<String> = HashSet::new();
214    let mut actions = Vec::new();
215
216    for entry in new_entries {
217        let etype = entry.get("type").and_then(Value::as_str).unwrap_or("");
218        if !STEM_TYPES.contains(&etype) {
219            actions.push(Action {
220                status: SKIP,
221                new_stem: String::new(),
222                old_seq: None,
223                old_stem: None,
224                reason: String::new(),
225                new_text: String::new(),
226            });
227            continue;
228        }
229
230        let new_stem = make_stem_name(entry);
231        let new_speaker = entry.get("speaker").and_then(Value::as_str);
232        let ekey = match_key(entry, strict);
233        let tkey = normalize_text(entry.get("text").and_then(Value::as_str), strict);
234
235        if let Some(m) = exact.get(&ekey) {
236            if !used_exact.contains(&ekey) {
237                used_exact.insert(ekey.clone());
238                used_text.insert(tkey.clone());
239                let old_seq = m.entry.get("seq").and_then(Value::as_i64);
240                actions.push(Action {
241                    status: if m.exists { COPY } else { MISSING },
242                    new_stem,
243                    old_seq,
244                    old_stem: Some(m.old_stem.clone()),
245                    reason: if m.exists {
246                        String::new()
247                    } else {
248                        "old stem file not on disk".into()
249                    },
250                    new_text: snip(entry.get("text").and_then(Value::as_str)),
251                });
252                continue;
253            }
254        }
255
256        if etype == "dialogue" {
257            if let Some(tm) = text_only.get(&tkey) {
258                if !used_text.contains(&tkey) {
259                    used_text.insert(tkey.clone());
260                    let old_speaker = tm.entry.get("speaker").and_then(Value::as_str);
261                    if old_speaker != new_speaker {
262                        actions.push(Action {
263                            status: SPEAKER,
264                            new_stem,
265                            old_seq: tm.entry.get("seq").and_then(Value::as_i64),
266                            old_stem: Some(tm.old_stem.clone()),
267                            reason: format!(
268                                "speaker: {} → {}",
269                                py_opt(old_speaker),
270                                py_opt(new_speaker)
271                            ),
272                            new_text: snip(entry.get("text").and_then(Value::as_str)),
273                        });
274                        continue;
275                    }
276                }
277            }
278        }
279
280        actions.push(Action {
281            status: NEW,
282            new_stem,
283            old_seq: None,
284            old_stem: None,
285            reason: "no matching old entry".into(),
286            new_text: snip(entry.get("text").and_then(Value::as_str)),
287        });
288    }
289    actions
290}
291
292/// How Python renders an optional speaker inside an f-string.
293fn py_opt(v: Option<&str>) -> String {
294    v.map(str::to_string).unwrap_or_else(|| "None".into())
295}
296
297/// Copy the COPY actions; return per-status counts.
298pub fn execute_migration(
299    actions: &[Action],
300    stems_dir: &Path,
301    dry_run: bool,
302) -> std::io::Result<HashMap<&'static str, usize>> {
303    let mut counts: HashMap<&'static str, usize> =
304        [(COPY, 0), (SPEAKER, 0), (NEW, 0), (MISSING, 0), (SKIP, 0)]
305            .into_iter()
306            .collect();
307    for a in actions {
308        *counts.entry(a.status).or_insert(0) += 1;
309        if a.status != COPY {
310            continue;
311        }
312        let Some(old) = &a.old_stem else { continue };
313        let (src, dst) = (stems_dir.join(old), stems_dir.join(&a.new_stem));
314        if src == dst || dry_run {
315            continue;
316        }
317        fs::copy(&src, &dst)?;
318    }
319    Ok(counts)
320}
321
322fn print_report(actions: &[Action], dry_run: bool) {
323    let label = if dry_run { "[DRY RUN] " } else { "" };
324    let stem_actions: Vec<&Action> = actions.iter().filter(|a| a.status != SKIP).collect();
325    log::info(&format!(
326        "\n{label}Migration plan ({} stem entries):\n",
327        stem_actions.len()
328    ));
329    for a in stem_actions {
330        match a.status {
331            COPY => {
332                if a.old_stem.as_deref() == Some(a.new_stem.as_str()) {
333                    log::info(&format!("  COPY     {}  (seq unchanged)", a.new_stem));
334                } else {
335                    log::info(&format!(
336                        "  COPY     {}  ← {}",
337                        a.new_stem,
338                        a.old_stem.clone().unwrap_or_default()
339                    ));
340                }
341            }
342            SPEAKER => log::info(&format!("  SPEAKER  {}  ({})", a.new_stem, a.reason)),
343            MISSING => log::info(&format!(
344                "  MISSING  {}  (matched seq {} but file absent)",
345                a.new_stem,
346                a.old_seq
347                    .map(|s| s.to_string())
348                    .unwrap_or_else(|| "None".into())
349            )),
350            NEW => log::info(&format!("  NEW      {}  ({})", a.new_stem, a.reason)),
351            _ => {}
352        }
353        if !a.new_text.is_empty() {
354            log::info(&format!("           \"{}\"", a.new_text));
355        }
356    }
357}
358
359fn print_summary(counts: &HashMap<&'static str, usize>, dry_run: bool) {
360    let g = |k: &str| *counts.get(k).unwrap_or(&0);
361    let need_gen = g(SPEAKER) + g(NEW) + g(MISSING);
362    let label = if dry_run { "[DRY RUN] " } else { "" };
363    log::info(&format!("\n{label}─── Summary ───"));
364    log::info(&format!(
365        "  COPY    : {:4}  (unchanged — reused, no TTS call)",
366        g(COPY)
367    ));
368    log::info(&format!(
369        "  SPEAKER : {:4}  (speaker changed → must regenerate)",
370        g(SPEAKER)
371    ));
372    log::info(&format!(
373        "  NEW     : {:4}  (no old match → must generate)",
374        g(NEW)
375    ));
376    log::info(&format!(
377        "  MISSING : {:4}  (old match but file absent → generate)",
378        g(MISSING)
379    ));
380    log::info(&format!(
381        "  SKIP    : {:4}  (non-stem entries, no action)",
382        g(SKIP)
383    ));
384    log::info("  ─────────────────────────────────────");
385    log::info(&format!("  Need generation : {need_gen}"));
386    log::info("");
387    if dry_run {
388        log::info("  Re-run without --dry-run to copy the COPY stems.");
389    } else {
390        log::info(&format!("  {} stems copied.", g(COPY)));
391    }
392    log::info("  Then run:  python XILP002_producer.py --episode <TAG>");
393    log::info("  XILP002 skips stems already on disk — only gaps get generated.");
394}
395
396fn entries_of(v: &Value) -> Vec<Entry> {
397    v.get("entries")
398        .and_then(Value::as_array)
399        .map(|a| a.iter().filter_map(|e| e.as_object().cloned()).collect())
400        .unwrap_or_default()
401}
402
403fn execute(a: &Args) -> anyhow::Result<i32> {
404    let (old_path, new_path, stems_dir) = match a.episode.as_ref().or(a.tag.as_ref()) {
405        Some(tag) => {
406            let slug = resolve_slug(a.show.as_deref(), "project.json");
407            let p = derive_paths(&slug, tag);
408            (
409                a.old.clone().unwrap_or_else(|| {
410                    workspace_root()
411                        .join("parsed")
412                        .join(format!("{}parsed_{slug}_{tag}.json", a.orig_prefix))
413                }),
414                a.new.clone().unwrap_or_else(|| p["parsed"].clone()),
415                a.stems.clone().unwrap_or_else(|| p["stems"].clone()),
416            )
417        }
418        None => match (&a.old, &a.new, &a.stems) {
419            (Some(o), Some(n), Some(s)) => (o.clone(), n.clone(), s.clone()),
420            _ => {
421                return Ok(parser_error(
422                    "Provide --episode, or all three of --old, --new, and --stems.",
423                ))
424            }
425        },
426    };
427
428    for (p, label) in [(&old_path, "--old"), (&new_path, "--new")] {
429        if !p.is_file() {
430            return Ok(parser_error(&format!(
431                "{label} file not found: {}",
432                p.display()
433            )));
434        }
435    }
436
437    log::info(&format!("  Old parsed : {}", old_path.display()));
438    log::info(&format!("  New parsed : {}", new_path.display()));
439    log::info(&format!("  Stems dir  : {}", stems_dir.display()));
440    log::info(&format!(
441        "  Match mode : {}",
442        if a.strict {
443            "strict"
444        } else {
445            "fuzzy (ignores em-dash / ellipsis variants)"
446        }
447    ));
448    log::info(&format!(
449        "  Dry run    : {}",
450        if a.dry_run { "True" } else { "False" }
451    ));
452
453    let old_data: Value = serde_json::from_str(&fs::read_to_string(&old_path)?)?;
454    let new_data: Value = serde_json::from_str(&fs::read_to_string(&new_path)?)?;
455    let actions = plan_migration(
456        &entries_of(&old_data),
457        &entries_of(&new_data),
458        &stems_dir,
459        a.strict,
460    );
461
462    if !a.quiet {
463        print_report(&actions, a.dry_run);
464    }
465    let counts = execute_migration(&actions, &stems_dir, a.dry_run)?;
466    print_summary(&counts, a.dry_run);
467    Ok(0)
468}
469
470pub fn run(args: &[OsString]) -> anyhow::Result<i32> {
471    log::init("migrate");
472    let _banner = banner::begin("XILP007 stem migrator", &super::argv_line(args));
473    let a: Args = match super::parse_or_exit("xil-migrate", args) {
474        Ok(a) => a,
475        Err(code) => return Ok(code),
476    };
477    execute(&a)
478}
479
480/// The clap definition behind `--help`, for man pages.
481pub fn command() -> clap::Command {
482    <Args as clap::CommandFactory>::command()
483}
484
485#[cfg(test)]
486mod tests {
487    use super::*;
488    use serde_json::json;
489
490    fn entries(v: Value) -> Vec<Entry> {
491        v.as_array()
492            .unwrap()
493            .iter()
494            .map(|e| e.as_object().unwrap().clone())
495            .collect()
496    }
497
498    #[test]
499    fn fuzzy_normalization_folds_punctuation() {
500        assert_eq!(normalize_text(Some("  a   b  "), false), "a b");
501        assert_eq!(normalize_text(Some("a \u{2014} b"), false), "a - b");
502        assert_eq!(normalize_text(Some("a\u{2026}"), false), "a...");
503        assert_eq!(
504            normalize_text(Some("\u{2018}q\u{2019} \u{201c}d\u{201d}"), false),
505            "'q' \"d\""
506        );
507        assert_eq!(
508            normalize_text(Some("a \u{2014} b"), true),
509            "a \u{2014} b",
510            "strict keeps the dash"
511        );
512        assert_eq!(normalize_text(None, false), "");
513    }
514
515    #[test]
516    fn stem_names_cover_dialogue_direction_and_preamble() {
517        let d = entries(
518            json!([{"seq": 19, "section": "act1", "scene": "scene-1", "type": "dialogue", "speaker": "maya"}]),
519        );
520        assert_eq!(make_stem_name(&d[0]), "019_act1-scene-1_maya.mp3");
521        let x = entries(json!([{"seq": 4, "section": "act1", "scene": null, "type": "direction"}]));
522        assert_eq!(make_stem_name(&x[0]), "004_act1_sfx.mp3");
523        let p = entries(
524            json!([{"seq": -2, "section": "preamble", "type": "dialogue", "speaker": "tina"}]),
525        );
526        assert_eq!(make_stem_name(&p[0]), "n002_preamble_tina.mp3");
527        let u = entries(json!([{"seq": 1, "section": null, "type": "dialogue", "speaker": "x"}]));
528        assert_eq!(make_stem_name(&u[0]), "001_unknown_x.mp3");
529    }
530
531    #[test]
532    fn plan_covers_copy_speaker_new_missing_and_skip() {
533        let tmp = tempfile::tempdir().unwrap();
534        let d = tmp.path();
535        fs::write(d.join("001_act1_adam.mp3"), "x").unwrap();
536        // 002's old file is deliberately absent → MISSING.
537
538        let old = entries(json!([
539            {"seq": 1, "type": "dialogue", "section": "act1", "speaker": "adam", "text": "Kept line."},
540            {"seq": 2, "type": "dialogue", "section": "act1", "speaker": "adam", "text": "Gone from disk."},
541            {"seq": 3, "type": "dialogue", "section": "act1", "speaker": "adam", "text": "Reassigned."}
542        ]));
543        let new = entries(json!([
544            {"seq": 1, "type": "section_header", "section": "act1", "text": "ACT ONE"},
545            {"seq": 2, "type": "dialogue", "section": "act1", "speaker": "adam", "text": "Kept line."},
546            {"seq": 3, "type": "dialogue", "section": "act1", "speaker": "adam", "text": "Gone from disk."},
547            {"seq": 4, "type": "dialogue", "section": "act1", "speaker": "maya", "text": "Reassigned."},
548            {"seq": 5, "type": "dialogue", "section": "act1", "speaker": "adam", "text": "Brand new."}
549        ]));
550
551        let plan = plan_migration(&old, &new, d, false);
552        let got: Vec<&str> = plan.iter().map(|a| a.status).collect();
553        assert_eq!(got, vec![SKIP, COPY, MISSING, SPEAKER, NEW]);
554        assert_eq!(plan[1].new_stem, "002_act1_adam.mp3");
555        assert_eq!(plan[1].old_stem.as_deref(), Some("001_act1_adam.mp3"));
556        assert_eq!(plan[3].reason, "speaker: adam → maya");
557        assert_eq!(plan[4].new_text, "Brand new.");
558    }
559
560    #[test]
561    fn fuzzy_matching_survives_a_punctuation_edit() {
562        let tmp = tempfile::tempdir().unwrap();
563        fs::write(tmp.path().join("001_act1_adam.mp3"), "x").unwrap();
564        let old = entries(
565            json!([{"seq": 1, "type": "dialogue", "section": "act1", "speaker": "adam", "text": "A \u{2014} B"}]),
566        );
567        let new = entries(
568            json!([{"seq": 1, "type": "dialogue", "section": "act1", "speaker": "adam", "text": "A - B"}]),
569        );
570        assert_eq!(
571            plan_migration(&old, &new, tmp.path(), false)[0].status,
572            COPY
573        );
574        assert_eq!(
575            plan_migration(&old, &new, tmp.path(), true)[0].status,
576            NEW,
577            "strict sees a different line"
578        );
579    }
580
581    #[test]
582    fn copy_only_runs_when_the_name_actually_changes() {
583        let tmp = tempfile::tempdir().unwrap();
584        let d = tmp.path();
585        fs::write(d.join("001_act1_adam.mp3"), "payload").unwrap();
586        let old = entries(
587            json!([{"seq": 1, "type": "dialogue", "section": "act1", "speaker": "adam", "text": "t"}]),
588        );
589        let new = entries(
590            json!([{"seq": 2, "type": "dialogue", "section": "act1", "speaker": "adam", "text": "t"}]),
591        );
592        let plan = plan_migration(&old, &new, d, false);
593        let counts = execute_migration(&plan, d, false).unwrap();
594        assert_eq!(counts[COPY], 1);
595        assert_eq!(
596            fs::read_to_string(d.join("002_act1_adam.mp3")).unwrap(),
597            "payload"
598        );
599
600        // A dry run touches nothing.
601        let plan2 = plan_migration(&old, &new, d, false);
602        fs::remove_file(d.join("002_act1_adam.mp3")).unwrap();
603        execute_migration(&plan2, d, true).unwrap();
604        assert!(!d.join("002_act1_adam.mp3").exists());
605    }
606}
xil/cmd/migrate.rs

xil/cmd/
migrate.rs
