xil_api/
gtts.rs

1//! Google Translate TTS, driven exactly as `gTTS` 2.5 drives it.
2//!
3//! Text is pre-processed and, past 100 characters, tokenised with gTTS's
4//! rules, then each chunk is posted to the `batchexecute` RPC and the
5//! base64 audio in each answer is concatenated. The rules use regex
6//! lookbehind, which the `regex` crate does not have, so they are written
7//! out by hand below — in the same order and with the same alternation
8//! priority as the compiled Python pattern.
9
10use serde_json::Value;
11
12use crate::{agent, base_url, call, read_bytes, ApiError};
13
14pub const DEFAULT_BASE_URL: &str = "https://translate.google.com";
15const MAX_CHARS: usize = 100;
16const RPC: &str = "jQ1olc";
17const USER_AGENT: &str =
18    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36";
19
20const TONE_MARKS: [char; 4] = ['?', '!', '？', '！'];
21const ABBREVIATIONS: [&str; 9] = ["dr", "jr", "mr", "mrs", "ms", "msgr", "prof", "sr", "st"];
22/// `symbols.ALL_PUNC`.
23const ALL_PUNC: &str = "?!？！.,¡()[]¿…‥،;:—。，、：\n";
24
25/// `pre_processors.tone_marks`: a space after every tone mark.
26fn pp_tone_marks(text: &str) -> String {
27    let mut out = String::with_capacity(text.len() + 8);
28    for c in text.chars() {
29        out.push(c);
30        if TONE_MARKS.contains(&c) {
31            out.push(' ');
32        }
33    }
34    out
35}
36
37/// `pre_processors.end_of_line`: drop `-\n`.
38fn pp_end_of_line(text: &str) -> String {
39    text.replace("-\n", "")
40}
41
42/// `pre_processors.abbreviations`: remove the period after each listed
43/// abbreviation (case-insensitive, no word boundary — gTTS's own regex),
44/// one abbreviation per pass, each pass judged on that pass's input.
45fn pp_abbreviations(text: &str) -> String {
46    let mut text = text.to_string();
47    for abbr in ABBREVIATIONS {
48        let chars: Vec<char> = text.chars().collect();
49        let a: Vec<char> = abbr.chars().collect();
50        let mut out = String::with_capacity(text.len());
51        for (i, &c) in chars.iter().enumerate() {
52            if c == '.' && i >= a.len() {
53                let before = &chars[i - a.len()..i];
54                if before
55                    .iter()
56                    .zip(&a)
57                    .all(|(x, y)| x.to_lowercase().eq(y.to_lowercase()))
58                {
59                    continue;
60                }
61            }
62            out.push(c);
63        }
64        text = out;
65    }
66    text
67}
68
69/// `pre_processors.word_sub`: `Esq.` → `Esquire`, case-insensitive.
70fn pp_word_sub(text: &str) -> String {
71    let lower = text.to_lowercase();
72    if lower.len() != text.len() {
73        // Non-ASCII case mapping changed byte offsets; fall back to char walk.
74        let chars: Vec<char> = text.chars().collect();
75        let mut out = String::new();
76        let mut i = 0;
77        while i < chars.len() {
78            let window: String = chars[i..(i + 4).min(chars.len())].iter().collect();
79            if window.to_lowercase() == "esq." {
80                out.push_str("Esquire");
81                i += 4;
82            } else {
83                out.push(chars[i]);
84                i += 1;
85            }
86        }
87        return out;
88    }
89    let mut out = String::new();
90    let mut i = 0;
91    while i < text.len() {
92        if lower[i..].starts_with("esq.") {
93            out.push_str("Esquire");
94            i += 4;
95        } else {
96            let c = text[i..].chars().next().unwrap();
97            out.push(c);
98            i += c.len_utf8();
99        }
100    }
101    out
102}
103
104fn is_other_punc(c: char) -> bool {
105    ALL_PUNC.contains(c) && !TONE_MARKS.contains(&c) && c != '.' && c != ',' && c != ':'
106}
107
108/// `Tokenizer([tone_marks, period_comma, colon, other_punctuation]).run`:
109/// `re.split` on the combined, case-insensitive pattern.
110fn tokenize(text: &str) -> Vec<String> {
111    let chars: Vec<char> = text.chars().collect();
112    let mut tokens = Vec::new();
113    let mut start = 0;
114    let mut i = 0;
115    while i < chars.len() {
116        let c = chars[i];
117        // (?<=[?!？！]). — any character but a newline after a tone mark.
118        let tone = i > 0 && TONE_MARKS.contains(&chars[i - 1]) && c != '\n';
119        // (?<!\.[a-z])[.,] followed by a space.
120        let period_comma = (c == '.' || c == ',')
121            && chars.get(i + 1) == Some(&' ')
122            && !(i >= 2 && chars[i - 2] == '.' && chars[i - 1].is_ascii_alphabetic());
123        // (?<!\d):
124        let colon = c == ':' && !(i > 0 && chars[i - 1].is_ascii_digit());
125        let len = if tone {
126            1
127        } else if period_comma {
128            2
129        } else if colon || is_other_punc(c) {
130            1
131        } else {
132            0
133        };
134        if len > 0 {
135            tokens.push(chars[start..i].iter().collect());
136            i += len;
137            start = i;
138        } else {
139            i += 1;
140        }
141    }
142    tokens.push(chars[start..].iter().collect());
143    tokens
144}
145
146/// `_ALL_PUNC_OR_SPACE.match(t)` — nothing but punctuation and ASCII whitespace.
147fn is_punc_or_space(t: &str) -> bool {
148    t.chars()
149        .all(|c| ALL_PUNC.contains(c) || " \t\n\r\x0b\x0c".contains(c))
150}
151
152/// Python `str.strip()`.
153fn py_strip(s: &str) -> String {
154    s.trim_matches(|c: char| c.is_whitespace() || ('\x1c'..='\x1f').contains(&c))
155        .to_string()
156}
157
158fn clean_tokens(tokens: Vec<String>) -> Vec<String> {
159    tokens
160        .into_iter()
161        .filter(|t| !is_punc_or_space(t))
162        .map(|t| py_strip(&t))
163        .collect()
164}
165
166/// `_minimize(the_string, " ", max_size)`.
167fn minimize(s: &str, max: usize) -> Vec<String> {
168    let mut out = Vec::new();
169    let mut chars: Vec<char> = s.chars().collect();
170    loop {
171        if chars.first() == Some(&' ') {
172            chars.remove(0);
173        }
174        if chars.len() <= max {
175            out.push(chars.iter().collect());
176            return out;
177        }
178        let idx = chars[..max].iter().rposition(|&c| c == ' ').unwrap_or(max);
179        out.push(chars[..idx].iter().collect());
180        chars = chars[idx..].to_vec();
181    }
182}
183
184/// `gTTS._tokenize(text)` — the chunks that become requests.
185pub fn text_parts(text: &str) -> Vec<String> {
186    let mut t = py_strip(text);
187    t = pp_tone_marks(&t);
188    t = pp_end_of_line(&t);
189    t = pp_abbreviations(&t);
190    t = pp_word_sub(&t);
191    if t.chars().count() <= MAX_CHARS {
192        return clean_tokens(vec![t]);
193    }
194    let tokens = clean_tokens(tokenize(&t));
195    tokens
196        .iter()
197        .flat_map(|tok| minimize(tok, MAX_CHARS))
198        .filter(|s| !s.is_empty())
199        .collect()
200}
201
202/// `urllib.parse.quote(s)` with the default `safe="/"`.
203fn quote(s: &str) -> String {
204    let mut out = String::new();
205    for b in s.bytes() {
206        match b {
207            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.' | b'-' | b'~' | b'/' => {
208                out.push(b as char)
209            }
210            _ => out.push_str(&format!("%{b:02X}")),
211        }
212    }
213    out
214}
215
216/// `json.dumps(obj, separators=(",", ":"))` — ASCII-escaped.
217fn compact_ascii(v: &Value) -> String {
218    let s = xil_core::pyjson::dumps(v, xil_core::pyjson::Style::COMPACT);
219    // pyjson's compact style uses ", " / ": "; gTTS asks for none.
220    reserialize_tight(&s)
221}
222
223/// Remove the spaces Python's default separators add, outside strings.
224fn reserialize_tight(s: &str) -> String {
225    let mut out = String::with_capacity(s.len());
226    let mut in_str = false;
227    let mut escaped = false;
228    let mut chars = s.chars().peekable();
229    while let Some(c) = chars.next() {
230        if in_str {
231            out.push(c);
232            if escaped {
233                escaped = false;
234            } else if c == '\\' {
235                escaped = true;
236            } else if c == '"' {
237                in_str = false;
238            }
239            continue;
240        }
241        match c {
242            '"' => {
243                in_str = true;
244                out.push(c);
245            }
246            ',' | ':' => {
247                out.push(c);
248                if chars.peek() == Some(&' ') {
249                    chars.next();
250                }
251            }
252            _ => out.push(c),
253        }
254    }
255    out
256}
257
258/// `gTTS._package_rpc(text)` → the form body.
259fn package_rpc(text: &str, lang: &str) -> String {
260    let parameter = serde_json::json!([text, lang, null, "null"]);
261    let escaped = compact_ascii(&parameter);
262    let rpc = serde_json::json!([[[RPC, escaped, null, "generic"]]]);
263    format!("f.req={}&", quote(&compact_ascii(&rpc)))
264}
265
266#[derive(Debug, thiserror::Error)]
267pub enum GttsError {
268    #[error("{0}")]
269    Api(#[from] ApiError),
270    #[error("200 (OK) from TTS API. Probable cause: Unknown")]
271    NoAudio,
272    #[error("No text to send to TTS API")]
273    NoText,
274}
275
276/// `gTTS(text=, lang=).save(path)` → the concatenated MP3 bytes.
277pub fn synthesize(text: &str, lang: &str) -> Result<Vec<u8>, GttsError> {
278    let parts = text_parts(text);
279    if parts.is_empty() {
280        return Err(GttsError::NoText);
281    }
282    let url = format!(
283        "{}/_/TranslateWebserverUi/data/batchexecute",
284        base_url("XIL_GTTS_BASE_URL", DEFAULT_BASE_URL)
285    );
286    let mut audio = Vec::new();
287    for part in parts {
288        let body = package_rpc(&part, lang);
289        let req = agent().post(&url).set("User-Agent", USER_AGENT).set(
290            "Content-Type",
291            "application/x-www-form-urlencoded;charset=utf-8",
292        );
293        let resp = call(req.send_string(&body))?;
294        let bytes = read_bytes(resp)?;
295        let text = String::from_utf8_lossy(&bytes);
296        for line in text.lines() {
297            if !line.contains(RPC) {
298                continue;
299            }
300            let needle = format!("{RPC}\",\"[\\\"");
301            let Some(pos) = line.find(&needle) else {
302                return Err(GttsError::NoAudio);
303            };
304            let rest = &line[pos + needle.len()..];
305            let Some(end) = rest.rfind("\\\"]") else {
306                return Err(GttsError::NoAudio);
307            };
308            audio.extend(decode_base64(&rest[..end]).ok_or(GttsError::NoAudio)?);
309        }
310    }
311    Ok(audio)
312}
313
314fn decode_base64(s: &str) -> Option<Vec<u8>> {
315    let mut out = Vec::with_capacity(s.len() * 3 / 4);
316    let mut buf = 0u32;
317    let mut bits = 0;
318    for b in s.bytes() {
319        let v = match b {
320            b'A'..=b'Z' => b - b'A',
321            b'a'..=b'z' => b - b'a' + 26,
322            b'0'..=b'9' => b - b'0' + 52,
323            b'+' => 62,
324            b'/' => 63,
325            b'=' => break,
326            _ => return None,
327        } as u32;
328        buf = (buf << 6) | v;
329        bits += 6;
330        if bits >= 8 {
331            bits -= 8;
332            out.push((buf >> bits) as u8);
333            buf &= (1 << bits) - 1;
334        }
335    }
336    Some(out)
337}
338
339#[cfg(test)]
340mod tests {
341    use super::*;
342
343    #[test]
344    fn short_text_is_one_part() {
345        assert_eq!(text_parts("  Hello there!  "), vec!["Hello there!"]);
346        assert!(text_parts(" ... ").is_empty());
347    }
348
349    #[test]
350    fn preprocessors() {
351        assert_eq!(
352            pp_abbreviations("Dr. Who and Mrs. Smith, first."),
353            "Dr Who and Mrs Smith, first"
354        );
355        assert_eq!(pp_word_sub("John Doe, esq."), "John Doe, Esquire");
356        assert_eq!(pp_tone_marks("Hi!Yes?"), "Hi! Yes? ");
357    }
358
359    #[test]
360    fn rpc_body_is_quoted_compact_json() {
361        let body = package_rpc("hello world", "en");
362        assert_eq!(
363            body,
364            "f.req=%5B%5B%5B%22jQ1olc%22%2C%22%5B%5C%22hello%20world%5C%22%2C%5C%22en%5C%22%2Cnull%2C%5C%22null%5C%22%5D%22%2Cnull%2C%22generic%22%5D%5D%5D&"
365        );
366    }
367
368    #[test]
369    fn base64_round_trip() {
370        assert_eq!(decode_base64("aGVsbG8=").unwrap(), b"hello");
371    }
372}
xil_api/gtts.rs

xil_api/
gtts.rs
