From d2a46c25219c21ac4f128da8512302935654d38e Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Mon, 27 Nov 2023 17:26:59 +0100
Subject: split code into several files

---
 src/datafiles.rs | 321 +++++++++++++++++++++++++++
 src/format.rs    | 349 +++++++++++++++++++++++++++++
 src/main.rs      | 664 +------------------------------------------------------
 3 files changed, 676 insertions(+), 658 deletions(-)
 create mode 100644 src/datafiles.rs
 create mode 100644 src/format.rs

diff --git a/src/datafiles.rs b/src/datafiles.rs
new file mode 100644
index 0000000..629badf
--- /dev/null
+++ b/src/datafiles.rs
@@ -0,0 +1,321 @@
+use std::collections::HashMap;
+use std::fs;
+use std::io::{self, BufRead};
+
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+
+use crate::charset::Charset;
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct Example {
+    pub ja: String,
+    pub en: String,
+    pub expl: String,
+    pub id: Option<String>,
+    pub chars: Charset,
+}
+
+// =====================================================================
+//                      PARSING DATA FILES
+// =====================================================================
+
+pub type DictIndex<'a> = HashMap<&'a str, Vec<roxmltree::Node<'a, 'a>>>;
+pub fn index_jmdict<'a>(dict: &'a roxmltree::Document) -> DictIndex<'a> {
+    let dict = dict
+        .root()
+        .children()
+        .find(|x| x.has_tag_name("JMdict"))
+        .unwrap();
+
+    let mut ret: DictIndex<'a> = HashMap::new();
+    for x in dict.children().filter(|x| x.has_tag_name("entry")) {
+        for r in x.children().filter(|x| x.has_tag_name("k_ele")) {
+            if let Some(keb) = r.children().find(|x| x.has_tag_name("keb")) {
+                let txt = keb.text().unwrap().trim();
+                ret.entry(txt).or_default().push(x);
+            }
+        }
+    }
+
+    ret
+}
+
+pub fn parse_kanjidic() -> Result<Vec<(String, Charset)>> {
+    let n3_kanji = Charset::new(&fs::read_to_string("data/n3_kanji.txt")?.trim());
+
+    let file = fs::read_to_string("data/kanjidic2.xml")?;
+    let xml = roxmltree::Document::parse(&file)?;
+    let kanjidic = xml.root().first_child().unwrap();
+    assert!(kanjidic.has_tag_name("kanjidic2"));
+
+    let mut levels = HashMap::new();
+
+    for x in kanjidic.children() {
+        if !x.has_tag_name("character") {
+            continue;
+        }
+        let mut literal = None;
+        let mut jlpt = None;
+        let mut grade = None;
+        for y in x.children() {
+            if y.has_tag_name("literal") {
+                literal = y.text();
+            }
+            if y.has_tag_name("misc") {
+                for z in y.children() {
+                    if z.has_tag_name("jlpt") {
+                        jlpt = z.text().and_then(|x| str::parse::<i32>(x).ok());
+                    }
+                    if z.has_tag_name("grade") {
+                        grade = z.text().and_then(|x| str::parse::<i32>(x).ok());
+                    }
+                }
+            }
+        }
+        match grade {
+            Some(i) if i <= 6 => grade = Some(7),
+            _ => (),
+        }
+        if let Some(lit) = literal {
+            assert_eq!(lit.chars().count(), 1);
+            let jlpt = match jlpt {
+                Some(4) => Some(5),
+                Some(3) => Some(4),
+                Some(2) if n3_kanji.contains(lit.chars().next().unwrap()) => Some(3),
+                x => x,
+            };
+            levels
+                .entry((jlpt, grade))
+                .or_insert(String::new())
+                .extend(lit.chars());
+        }
+    }
+
+    let mut levels = levels.into_iter().collect::<Vec<_>>();
+    levels.sort_by_key(|((j, g), _)| match (j, g) {
+        (Some(j), Some(g)) => (10 - *j) * 20 + *g,
+        (Some(j), None) => (10 - *j) * 20 + 15,
+        (None, Some(g)) => 1000 + *g,
+        (None, None) => 1015,
+    });
+
+    let mut ret = Vec::new();
+    let mut pc = Charset::default();
+    for ((j, g), chars) in levels.into_iter() {
+        let name = match (j, g) {
+            (Some(j), Some(7)) => format!("N{}a", j),
+            (Some(j), Some(8)) => format!("N{}b", j),
+            (Some(j), Some(g)) => format!("N{}-{}", j, g),
+            (Some(j), None) => format!("N{}+", j),
+            (None, Some(7)) => format!("N0a"),
+            (None, Some(8)) => format!("N0b"),
+            (None, Some(g)) => format!("N0-{}", g),
+            (None, None) => format!("N0+"),
+        };
+        let chars = Charset::new(chars).diff(&pc);
+        pc = pc.union(&chars);
+        ret.push((name, chars));
+    }
+
+    Ok(ret)
+}
+
+pub fn read_kanji_levels() -> Result<Vec<(String, String)>> {
+    Ok(fs::read_to_string("data/kanji_levels.txt")?
+        .lines()
+        .filter_map(|l| l.split_once(": "))
+        .map(|(l, k)| (l.to_string(), k.to_string()))
+        .collect::<Vec<_>>())
+}
+
+pub fn read_examples(all_kanji: &Charset) -> Result<Vec<Example>> {
+    let file = fs::File::open("data/examples.utf")?;
+
+    let mut ret = Vec::new();
+    let mut a = "".to_string();
+
+    for (i, line) in io::BufReader::new(file).lines().enumerate() {
+        let line = line?;
+        if line.starts_with("A:") {
+            a = line;
+        } else if line.starts_with("B:") {
+            let s = a.strip_prefix("A: ");
+            let t = line.strip_prefix("B: ");
+            if let (Some(a), Some(b)) = (s, t) {
+                if let Some((ja, eng)) = a.split_once("\t") {
+                    if let Some((eng, id)) = eng.split_once("#") {
+                        ret.push(Example {
+                            ja: ja.to_string(),
+                            en: eng.to_string(),
+                            expl: b.to_string(),
+                            id: Some(id.to_string()),
+                            chars: Charset::new(ja).inter(all_kanji),
+                        });
+                    } else {
+                        ret.push(Example {
+                            ja: ja.to_string(),
+                            en: eng.to_string(),
+                            expl: b.to_string(),
+                            id: None,
+                            chars: Charset::new(ja).inter(all_kanji),
+                        });
+                    }
+                }
+            }
+        }
+        if i % 10000 == 0 {
+            eprintln!("read examples: {}/300 (x1000)", i / 1000);
+        }
+    }
+
+    Ok(ret)
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+pub struct JlptVocab {
+    pub level: String,
+    pub chars: Charset,
+    pub kanji: String,
+    pub kana: String,
+    pub en: String,
+}
+
+impl JlptVocab {
+    pub fn to_string(&self) -> String {
+        format!(
+            "{}\t{}\t{}\t{}\t{}",
+            self.level,
+            self.chars.to_string(),
+            self.kanji,
+            self.kana,
+            self.en
+        )
+    }
+}
+
+pub fn parse_jlpt_vocab(all_kanji: &Charset) -> Result<()> {
+    let mut vocab = vec![];
+    vocab.extend(parse_jlpt_vocab_combined(
+        "data/n5_vocab.txt",
+        "N5",
+        all_kanji,
+    )?);
+    vocab.extend(parse_jlpt_vocab_split(
+        "data/n4_vocab_hiragana.txt",
+        "data/n4_vocab_eng.txt",
+        "N4",
+        all_kanji,
+    )?);
+    vocab.extend(parse_jlpt_vocab_split(
+        "data/n3_vocab_hiragana.txt",
+        "data/n3_vocab_eng.txt",
+        "N3",
+        all_kanji,
+    )?);
+    vocab.extend(parse_jlpt_vocab_split(
+        "data/n2_vocab_hiragana.txt",
+        "data/n2_vocab_eng.txt",
+        "N2",
+        all_kanji,
+    )?);
+    vocab.extend(parse_jlpt_vocab_split(
+        "data/n1_vocab_hiragana.txt",
+        "data/n1_vocab_eng.txt",
+        "N1",
+        all_kanji,
+    )?);
+    for v in vocab.iter() {
+        println!("{}", v.to_string());
+    }
+    Ok(())
+}
+
+fn parse_jlpt_vocab_combined(
+    file: &str,
+    level: &str,
+    all_kanji: &Charset,
+) -> Result<Vec<JlptVocab>> {
+    let lines = jlpt_vocab_read_file(file)?;
+    let mut ret = vec![];
+    for (kanji, answer) in lines {
+        let (eng, kana) = match answer.split_once('\n') {
+            Some((a, b)) => (a, b.trim()),
+            None => (answer.trim(), ""),
+        };
+        for kanji in kanji.split('/') {
+            ret.push(JlptVocab {
+                level: level.to_string(),
+                chars: Charset::new(kanji).inter(all_kanji),
+                kanji: kanji.to_string(),
+                kana: kana.to_string(),
+                en: eng.to_string(),
+            });
+        }
+    }
+    Ok(ret)
+}
+
+fn parse_jlpt_vocab_split(
+    kana_file: &str,
+    eng_file: &str,
+    level: &str,
+    all_kanji: &Charset,
+) -> Result<Vec<JlptVocab>> {
+    let eng_lines = jlpt_vocab_read_file(eng_file)?
+        .into_iter()
+        .collect::<HashMap<String, String>>();
+
+    let lines = jlpt_vocab_read_file(kana_file)?;
+    let mut ret = vec![];
+    for (kanji, kana) in lines {
+        let eng = eng_lines.get(&kanji).or(eng_lines.get(&kana));
+        if let Some(eng) = eng {
+            for kanji in kanji.split('/') {
+                ret.push(JlptVocab {
+                    level: level.to_string(),
+                    chars: Charset::new(kanji).inter(all_kanji),
+                    kanji: kanji.to_string(),
+                    kana: kana.to_string(),
+                    en: eng.to_string(),
+                });
+            }
+        }
+    }
+    Ok(ret)
+}
+
+fn jlpt_vocab_read_file(file: &str) -> Result<Vec<(String, String)>> {
+    let re = regex::Regex::new(r#"<span class="\w+">"#)?;
+
+    let file = fs::File::open(file)?;
+    let mut ret = vec![];
+    for line in io::BufReader::new(file).lines() {
+        let line = line?.replace("<br>", "\n").replace("</span>", "");
+        let line = re.replace_all(&line, "");
+        if let Some((a, b)) = line.split_once('|') {
+            ret.push((a.trim().to_string(), b.trim().to_string()));
+        }
+    }
+
+    Ok(ret)
+}
+
+pub fn load_jlpt_vocab() -> Result<Vec<JlptVocab>> {
+    let file = fs::File::open("data/jlpt_vocab.txt")?;
+    let mut ret = vec![];
+    for line in io::BufReader::new(file).lines() {
+        let line = line?;
+        let line = line.splitn(5, "\t").collect::<Vec<_>>();
+        if line.len() == 5 {
+            ret.push(JlptVocab {
+                level: line[0].to_string(),
+                chars: Charset::new(line[1]),
+                kanji: line[2].to_string(),
+                kana: line[3].to_string(),
+                en: line[4].to_string(),
+            });
+        }
+    }
+    Ok(ret)
+}
diff --git a/src/format.rs b/src/format.rs
new file mode 100644
index 0000000..1cdde1b
--- /dev/null
+++ b/src/format.rs
@@ -0,0 +1,349 @@
+use std::fs;
+
+use anyhow::Result;
+
+use crate::charset::Charset;
+use crate::*;
+
+// =====================================================================
+//                          FORMATTING TO HTML
+// =====================================================================
+
+pub fn format_batch<'a>(dict_idx: &DictIndex<'a>, count: usize, (i, batch): (usize, &Batch)) {
+    format_batch_aux(dict_idx, count, i, batch).expect("format batch");
+}
+
+fn format_batch_aux<'a>(
+    dict_idx: &DictIndex<'a>,
+    count: usize,
+    i: usize,
+    batch: &Batch,
+) -> Result<()> {
+    let mut f = io::BufWriter::new(fs::File::create(format!("public/{:03}.html", i))?);
+    write!(
+        f,
+        r#"<!DOCTYPE html>
+        <html>
+            <head>
+                <meta charset=\"UTF-8\" />
+                <title>Batch #{:03}</title>
+                <link rel="stylesheet" type="text/css" href="style.css" />
+            </head>
+            <body><div class="batch_page">"#,
+        i
+    )?;
+
+    writeln!(f, r#"<p><a href="index.html">index</a>"#)?;
+    for j in 0..count {
+        if j != i {
+            writeln!(f, r#" <a href="{:03}.html">{:03}</a>"#, j, j)?;
+        } else {
+            writeln!(f, " {:03}", j)?;
+        }
+    }
+    writeln!(f, r#"</p>"#)?;
+    writeln!(f, "<p>Level: {}</p>", batch.level)?;
+
+    write!(f, r#"<p class="ja">"#)?;
+    let mut ex_prev = Charset::default();
+    for ex in batch.examples.iter() {
+        let ex_chars = ex.chars.inter(&batch.chars);
+        for c in ex_chars.diff(&ex_prev).chars().iter() {
+            write!(
+                f,
+                r#"<a href="https://jisho.org/search/{}%20%23kanji">{}</a>"#,
+                c, c
+            )?;
+        }
+        ex_prev = ex_prev.union(&ex_chars);
+    }
+    writeln!(f, r#"</p>"#)?;
+
+    for ex in batch.examples.iter() {
+        writeln!(f, "<hr />")?;
+        write!(f, r#"<p class="ja">"#)?;
+        for c in ex.ja.chars() {
+            if batch.chars.contains(c) {
+                write!(f, r#"<span class="char_cur">{}</span>"#, c)?;
+            } else if batch.chars_p1.contains(c) {
+                write!(f, r#"<span class="char_p1">{}</span>"#, c)?;
+            } else if batch.chars_p2.contains(c) {
+                write!(f, r#"<span class="char_p2">{}</span>"#, c)?;
+            } else if batch.chars_bad.contains(c) {
+                write!(f, r#"<span class="char_bad">{}</span>"#, c)?;
+            } else {
+                write!(f, "{}", c)?;
+            }
+        }
+        writeln!(f, "</p>")?;
+        writeln!(f, r#"<p class="en">{}</p>"#, ex.en)?;
+
+        writeln!(f, r#"<details><summary>Explanation</summary>"#)?;
+        let mut expl_batch = Vec::new();
+        let mut expl_all = Vec::new();
+        for word in ex.expl.split(|c| c == ' ' || c == '~') {
+            let (keb, reb) = expl_clean_word(word);
+            let wchars = Charset::new(keb);
+            if !wchars.intersects(&ex.chars) {
+                continue;
+            }
+            if let Some(ents) = dict_idx.get(keb) {
+                for ent in ents.iter() {
+                    if let Some(s) = dict_str(keb, reb, ent) {
+                        if wchars.intersects(&batch.chars) {
+                            expl_batch.push(s);
+                        } else {
+                            expl_all.push(s);
+                        }
+                    }
+                }
+            }
+        }
+        for be in expl_batch {
+            writeln!(f, r#"<p>{}</p>"#, be)?;
+        }
+        writeln!(f, r#"<p class="chars">"#)?;
+        for c in ex.chars.inter(&batch.chars).chars().iter() {
+            writeln!(
+                f,
+                r#"<a href="https://jisho.org/search/{}%20%23kanji">{}</a>"#,
+                c, c
+            )?;
+        }
+        writeln!(f, r#"</p>"#)?;
+        for be in expl_all {
+            writeln!(f, r#"<p>{}</p>"#, be)?;
+        }
+        writeln!(f, r#"</details>"#)?;
+    }
+
+    writeln!(f, "<hr />")?;
+    format_vocab(
+        &mut f,
+        &batch
+            .extra_vocab
+            .iter()
+            .filter(|v| batch.level.contains(&v.level))
+            .collect::<Vec<_>>(),
+        "Extra vocabulary (this level)",
+    )?;
+    format_vocab(
+        &mut f,
+        &batch
+            .extra_vocab
+            .iter()
+            .filter(|v| !batch.level.contains(&v.level))
+            .collect::<Vec<_>>(),
+        "Extra vocabulary (previous levels)",
+    )?;
+
+    writeln!(
+        f,
+        r#"<details><summary>Extra examples (reading practice)</summary><table class="extratable">"#
+    )?;
+    for ex in batch.extra_examples.iter() {
+        let mut expl1 = Vec::new();
+        let mut expl2 = Vec::new();
+        for word in ex.expl.split(|c| c == ' ' || c == '~') {
+            let (keb, reb) = expl_clean_word(word);
+            let wchars = Charset::new(keb);
+            if !wchars.intersects(&ex.chars) {
+                continue;
+            }
+            if let Some(ents) = dict_idx.get(keb) {
+                for ent in ents.iter() {
+                    if let Some(s) = dict_str_short(keb, reb, ent) {
+                        if wchars.intersects(&batch.chars) {
+                            expl1.push(s);
+                        } else {
+                            expl2.push(s);
+                        }
+                    }
+                }
+            }
+        }
+        expl1.extend(expl2.into_iter());
+        let expl = expl1.join("<br />");
+        writeln!(
+            f,
+            r#"<tr><td><details><summary class="tab_large2 font_ja">&nbsp;&nbsp;{}&nbsp;&nbsp;</summary><div style="text-align: center">{}<br />{}</div></details></td></tr>"#,
+            ex.ja, ex.en, expl
+        )?;
+    }
+    writeln!(f, r#"</table></details>"#)?;
+
+    writeln!(f, "<hr />")?;
+    writeln!(f, "<p>＼(≧▽≦)／</p>")?;
+
+    write!(f, "<div></body></html>")?;
+    f.flush()?;
+    Ok(())
+}
+
+fn format_vocab(f: &mut impl Write, vocab: &[&JlptVocab], t: &str) -> Result<()> {
+    if !vocab.is_empty() {
+        writeln!(
+            f,
+            r#"<details><summary>{}</summary><table class="vocabtable">"#,
+            t
+        )?;
+        for v in vocab {
+            writeln!(
+                f,
+                r#"<tr><td>{}</td><td>&nbsp;&nbsp;<span class="tab_large font_ja">{}</span>&nbsp;&nbsp;</td><td>{}</td><td class="font_ja">{}</td></tr>"#,
+                v.level, v.kanji, v.en, v.kana
+            )?;
+        }
+        writeln!(f, "</table></details>")?;
+    }
+    Ok(())
+}
+
+fn expl_clean_word(w: &str) -> (&str, Option<&str>) {
+    let mut ret = w;
+    for delim in ['(', '{', '['] {
+        if let Some((s, _)) = ret.split_once(delim) {
+            ret = s;
+        }
+    }
+    let p = w
+        .split_once('(')
+        .and_then(|(_, r)| r.split_once(')'))
+        .map(|(p, _)| p);
+    (ret, p)
+}
+
+fn dict_str_short<'a>(
+    qkeb: &str,
+    qreb: Option<&str>,
+    ent: &roxmltree::Node<'a, 'a>,
+) -> Option<String> {
+    let r_ele = ent.children().find(|x| x.has_tag_name("r_ele")).unwrap();
+    let reb = r_ele.children().find(|x| x.has_tag_name("reb")).unwrap();
+    let reb = reb.text().unwrap().trim();
+
+    if qreb.map(|x| x != reb).unwrap_or(false) {
+        return None;
+    }
+
+    Some(format!(
+        r#"<span class="font_ja">{} 【{}】</span>"#,
+        qkeb, reb
+    ))
+}
+
+fn dict_str<'a>(qkeb: &str, qreb: Option<&str>, ent: &roxmltree::Node<'a, 'a>) -> Option<String> {
+    let mut ret = dict_str_short(qkeb, qreb, ent)?;
+
+    for sense in ent.children().filter(|x| x.has_tag_name("sense")) {
+        if let Some(s) = sense.children().find(|x| x.has_tag_name("gloss")) {
+            ret.extend(format!(" {};", s.text().unwrap().trim()).chars());
+        }
+    }
+
+    if ret.chars().rev().next() == Some(';') {
+        ret.pop();
+    }
+    Some(ret)
+}
+
+pub fn format_index(batches: &[Batch], kanji_levels: &[(String, String)]) -> Result<()> {
+    let mut f = io::BufWriter::new(fs::File::create("public/index.html")?);
+    write!(
+        f,
+        r#"<!DOCTYPE html>
+        <html>
+            <head>
+                <meta charset=\"UTF-8\" />
+                <title>List of batches</title>
+                <link rel="stylesheet" type="text/css" href="style.css" />
+            </head>
+            <body><div class="index_page">"#
+    )?;
+
+    writeln!(f, r#"<p><a href="about.html">About / How-to</a></p><hr />"#)?;
+
+    writeln!(f, "<table>")?;
+    writeln!(f, "<tr><th>Num</th><th>Level</th><th>Kanji</th><th>Examples</th><th>Lesson-1</th><th>Lesson-2</th><th>Ignore</th></tr>")?;
+    for (i, batch) in batches.iter().enumerate() {
+        writeln!(
+            f,
+            r#"<tr><td><a href="{:03}.html">{:03}</a></td><td>{}</td><td class="font_ja">{}</td><td>&nbsp;&nbsp;{}</td><td class="font_ja">{}</td><td class="font_ja">{}</td><td class="font_ja">{}</td></tr>"#,
+            i,
+            i,
+            batch.level,
+            batch.chars.to_string(),
+            batch.examples.len(),
+            batch.chars_p1.to_string(),
+            batch.chars_p2.to_string(),
+            batch.chars_bad.to_string()
+        )?;
+    }
+    writeln!(f, r#"</table>"#)?;
+
+    writeln!(f, "<hr />")?;
+
+    let all_chars = Charset::from_iter(
+        batches
+            .iter()
+            .map(|x| x.chars.chars().iter().copied())
+            .flatten(),
+    );
+    writeln!(f, "<table>")?;
+    writeln!(
+        f,
+        r#"<tr><th>Level</th><th>Count</th><th width="60%">Kanji</th><th>Missing kanji</th></tr>"#
+    )?;
+    for (lvl, chars) in kanji_levels.iter() {
+        if lvl == "N0+" || lvl.ends_with("-10") {
+            continue;
+        }
+        let chars = Charset::new(chars);
+        let missing = chars.diff(&all_chars);
+        writeln!(
+            f,
+            r#"<tr><td>{}</td><td>{}</td><td class="font_ja">{}</td><td><span class="font_ja">{}</span> ({})</td></tr>"#,
+            lvl,
+            chars.len(),
+            chars.to_string(),
+            missing.to_string(),
+            missing.len()
+        )?;
+    }
+    writeln!(f, "</table>")?;
+
+    write!(f, "</div></body></html>")?;
+    f.flush()?;
+    Ok(())
+}
+
+pub fn format_about() -> Result<()> {
+    let mut f = io::BufWriter::new(fs::File::create("public/about.html")?);
+    write!(
+        f,
+        r#"<!DOCTYPE html>
+        <html>
+            <head>
+                <meta charset=\"UTF-8\" />
+                <title>Datagengo README</title>
+                <link rel="stylesheet" type="text/css" href="style.css" />
+            </head>
+            <body>"#
+    )?;
+
+    writeln!(f, r#"<div class="about_page">"#)?;
+    writeln!(
+        f,
+        r#"<p><a href="index.html">Back to lessons</a></p><hr />"#
+    )?;
+
+    writeln!(
+        f,
+        "{}",
+        markdown::to_html(&fs::read_to_string("README.md")?)
+    )?;
+
+    writeln!(f, r#"</div></body></html>"#)?;
+
+    Ok(())
+}
diff --git a/src/main.rs b/src/main.rs
index 4ec20d0..5d6b7d7 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,15 +1,19 @@
 use std::collections::HashMap;
 use std::fs;
-use std::io::{self, BufRead, Write};
+use std::io::{self, Write};
 
-use anyhow::Result;
+//use anyhow::Result;
 use rand::prelude::*;
 use rayon::prelude::*;
 use serde::{Deserialize, Serialize};
 use structopt::StructOpt;
 
 mod charset;
+mod datafiles;
+mod format;
 use charset::Charset;
+use datafiles::*;
+use format::*;
 
 #[derive(Debug, StructOpt)]
 #[structopt(name = "datagengo", about = "Japanese example practice maker")]
@@ -194,310 +198,6 @@ fn main() {
     }
 }
 
-// =====================================================================
-//                      PARSING DATA FILES
-// =====================================================================
-
-type DictIndex<'a> = HashMap<&'a str, Vec<roxmltree::Node<'a, 'a>>>;
-fn index_jmdict<'a>(dict: &'a roxmltree::Document) -> DictIndex<'a> {
-    let dict = dict
-        .root()
-        .children()
-        .find(|x| x.has_tag_name("JMdict"))
-        .unwrap();
-
-    let mut ret: DictIndex<'a> = HashMap::new();
-    for x in dict.children().filter(|x| x.has_tag_name("entry")) {
-        for r in x.children().filter(|x| x.has_tag_name("k_ele")) {
-            if let Some(keb) = r.children().find(|x| x.has_tag_name("keb")) {
-                let txt = keb.text().unwrap().trim();
-                ret.entry(txt).or_default().push(x);
-            }
-        }
-    }
-
-    ret
-}
-
-fn parse_kanjidic() -> Result<Vec<(String, Charset)>> {
-    let n3_kanji = Charset::new(&fs::read_to_string("data/n3_kanji.txt")?.trim());
-
-    let file = fs::read_to_string("data/kanjidic2.xml")?;
-    let xml = roxmltree::Document::parse(&file)?;
-    let kanjidic = xml.root().first_child().unwrap();
-    assert!(kanjidic.has_tag_name("kanjidic2"));
-
-    let mut levels = HashMap::new();
-
-    for x in kanjidic.children() {
-        if !x.has_tag_name("character") {
-            continue;
-        }
-        let mut literal = None;
-        let mut jlpt = None;
-        let mut grade = None;
-        for y in x.children() {
-            if y.has_tag_name("literal") {
-                literal = y.text();
-            }
-            if y.has_tag_name("misc") {
-                for z in y.children() {
-                    if z.has_tag_name("jlpt") {
-                        jlpt = z.text().and_then(|x| str::parse::<i32>(x).ok());
-                    }
-                    if z.has_tag_name("grade") {
-                        grade = z.text().and_then(|x| str::parse::<i32>(x).ok());
-                    }
-                }
-            }
-        }
-        match grade {
-            Some(i) if i <= 6 => grade = Some(7),
-            _ => (),
-        }
-        if let Some(lit) = literal {
-            assert_eq!(lit.chars().count(), 1);
-            let jlpt = match jlpt {
-                Some(4) => Some(5),
-                Some(3) => Some(4),
-                Some(2) if n3_kanji.contains(lit.chars().next().unwrap()) => Some(3),
-                x => x,
-            };
-            levels
-                .entry((jlpt, grade))
-                .or_insert(String::new())
-                .extend(lit.chars());
-        }
-    }
-
-    let mut levels = levels.into_iter().collect::<Vec<_>>();
-    levels.sort_by_key(|((j, g), _)| match (j, g) {
-        (Some(j), Some(g)) => (10 - *j) * 20 + *g,
-        (Some(j), None) => (10 - *j) * 20 + 15,
-        (None, Some(g)) => 1000 + *g,
-        (None, None) => 1015,
-    });
-
-    let mut ret = Vec::new();
-    let mut pc = Charset::default();
-    for ((j, g), chars) in levels.into_iter() {
-        let name = match (j, g) {
-            (Some(j), Some(7)) => format!("N{}a", j),
-            (Some(j), Some(8)) => format!("N{}b", j),
-            (Some(j), Some(g)) => format!("N{}-{}", j, g),
-            (Some(j), None) => format!("N{}+", j),
-            (None, Some(7)) => format!("N0a"),
-            (None, Some(8)) => format!("N0b"),
-            (None, Some(g)) => format!("N0-{}", g),
-            (None, None) => format!("N0+"),
-        };
-        let chars = Charset::new(chars).diff(&pc);
-        pc = pc.union(&chars);
-        ret.push((name, chars));
-    }
-
-    Ok(ret)
-}
-
-fn read_kanji_levels() -> Result<Vec<(String, String)>> {
-    Ok(fs::read_to_string("data/kanji_levels.txt")?
-        .lines()
-        .filter_map(|l| l.split_once(": "))
-        .map(|(l, k)| (l.to_string(), k.to_string()))
-        .collect::<Vec<_>>())
-}
-
-fn read_examples(all_kanji: &Charset) -> Result<Vec<Example>> {
-    let file = fs::File::open("data/examples.utf")?;
-
-    let mut ret = Vec::new();
-    let mut a = "".to_string();
-
-    for (i, line) in io::BufReader::new(file).lines().enumerate() {
-        let line = line?;
-        if line.starts_with("A:") {
-            a = line;
-        } else if line.starts_with("B:") {
-            let s = a.strip_prefix("A: ");
-            let t = line.strip_prefix("B: ");
-            if let (Some(a), Some(b)) = (s, t) {
-                if let Some((ja, eng)) = a.split_once("\t") {
-                    if let Some((eng, id)) = eng.split_once("#") {
-                        ret.push(Example {
-                            ja: ja.to_string(),
-                            en: eng.to_string(),
-                            expl: b.to_string(),
-                            id: Some(id.to_string()),
-                            chars: Charset::new(ja).inter(all_kanji),
-                        });
-                    } else {
-                        ret.push(Example {
-                            ja: ja.to_string(),
-                            en: eng.to_string(),
-                            expl: b.to_string(),
-                            id: None,
-                            chars: Charset::new(ja).inter(all_kanji),
-                        });
-                    }
-                }
-            }
-        }
-        if i % 10000 == 0 {
-            eprintln!("read examples: {}/300 (x1000)", i / 1000);
-        }
-    }
-
-    Ok(ret)
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
-struct JlptVocab {
-    level: String,
-    chars: Charset,
-    kanji: String,
-    kana: String,
-    en: String,
-}
-
-impl JlptVocab {
-    fn to_string(&self) -> String {
-        format!(
-            "{}\t{}\t{}\t{}\t{}",
-            self.level,
-            self.chars.to_string(),
-            self.kanji,
-            self.kana,
-            self.en
-        )
-    }
-}
-
-fn parse_jlpt_vocab(all_kanji: &Charset) -> Result<()> {
-    let mut vocab = vec![];
-    vocab.extend(parse_jlpt_vocab_combined(
-        "data/n5_vocab.txt",
-        "N5",
-        all_kanji,
-    )?);
-    vocab.extend(parse_jlpt_vocab_split(
-        "data/n4_vocab_hiragana.txt",
-        "data/n4_vocab_eng.txt",
-        "N4",
-        all_kanji,
-    )?);
-    vocab.extend(parse_jlpt_vocab_split(
-        "data/n3_vocab_hiragana.txt",
-        "data/n3_vocab_eng.txt",
-        "N3",
-        all_kanji,
-    )?);
-    vocab.extend(parse_jlpt_vocab_split(
-        "data/n2_vocab_hiragana.txt",
-        "data/n2_vocab_eng.txt",
-        "N2",
-        all_kanji,
-    )?);
-    vocab.extend(parse_jlpt_vocab_split(
-        "data/n1_vocab_hiragana.txt",
-        "data/n1_vocab_eng.txt",
-        "N1",
-        all_kanji,
-    )?);
-    for v in vocab.iter() {
-        println!("{}", v.to_string());
-    }
-    Ok(())
-}
-
-fn parse_jlpt_vocab_combined(
-    file: &str,
-    level: &str,
-    all_kanji: &Charset,
-) -> Result<Vec<JlptVocab>> {
-    let lines = jlpt_vocab_read_file(file)?;
-    let mut ret = vec![];
-    for (kanji, answer) in lines {
-        let (eng, kana) = match answer.split_once('\n') {
-            Some((a, b)) => (a, b.trim()),
-            None => (answer.trim(), ""),
-        };
-        for kanji in kanji.split('/') {
-            ret.push(JlptVocab {
-                level: level.to_string(),
-                chars: Charset::new(kanji).inter(all_kanji),
-                kanji: kanji.to_string(),
-                kana: kana.to_string(),
-                en: eng.to_string(),
-            });
-        }
-    }
-    Ok(ret)
-}
-
-fn parse_jlpt_vocab_split(
-    kana_file: &str,
-    eng_file: &str,
-    level: &str,
-    all_kanji: &Charset,
-) -> Result<Vec<JlptVocab>> {
-    let eng_lines = jlpt_vocab_read_file(eng_file)?
-        .into_iter()
-        .collect::<HashMap<String, String>>();
-
-    let lines = jlpt_vocab_read_file(kana_file)?;
-    let mut ret = vec![];
-    for (kanji, kana) in lines {
-        let eng = eng_lines.get(&kanji).or(eng_lines.get(&kana));
-        if let Some(eng) = eng {
-            for kanji in kanji.split('/') {
-                ret.push(JlptVocab {
-                    level: level.to_string(),
-                    chars: Charset::new(kanji).inter(all_kanji),
-                    kanji: kanji.to_string(),
-                    kana: kana.to_string(),
-                    en: eng.to_string(),
-                });
-            }
-        }
-    }
-    Ok(ret)
-}
-
-fn jlpt_vocab_read_file(file: &str) -> Result<Vec<(String, String)>> {
-    let re = regex::Regex::new(r#"<span class="\w+">"#)?;
-
-    let file = fs::File::open(file)?;
-    let mut ret = vec![];
-    for line in io::BufReader::new(file).lines() {
-        let line = line?.replace("<br>", "\n").replace("</span>", "");
-        let line = re.replace_all(&line, "");
-        if let Some((a, b)) = line.split_once('|') {
-            ret.push((a.trim().to_string(), b.trim().to_string()));
-        }
-    }
-
-    Ok(ret)
-}
-
-fn load_jlpt_vocab() -> Result<Vec<JlptVocab>> {
-    let file = fs::File::open("data/jlpt_vocab.txt")?;
-    let mut ret = vec![];
-    for line in io::BufReader::new(file).lines() {
-        let line = line?;
-        let line = line.splitn(5, "\t").collect::<Vec<_>>();
-        if line.len() == 5 {
-            ret.push(JlptVocab {
-                level: line[0].to_string(),
-                chars: Charset::new(line[1]),
-                kanji: line[2].to_string(),
-                kana: line[3].to_string(),
-                en: line[4].to_string(),
-            });
-        }
-    }
-    Ok(ret)
-}
-
 // =====================================================================
 //                      BATCH STRUCTURES AND GENERATION
 // =====================================================================
@@ -505,15 +205,6 @@ fn load_jlpt_vocab() -> Result<Vec<JlptVocab>> {
 const CHARS_PER_BATCH: usize = 20;
 const MAX_NEW_CHARS_PER_EX: usize = 5;
 
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
-struct Example {
-    ja: String,
-    en: String,
-    expl: String,
-    id: Option<String>,
-    chars: Charset,
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)]
 struct Batch {
     level: String,
@@ -1109,346 +800,3 @@ fn add_extra_examples(all_batches: &mut [Batch], examples: &[Example]) {
         );
     }
 }
-
-// =====================================================================
-//                          FORMATTING TO HTML
-// =====================================================================
-
-fn format_batch<'a>(dict_idx: &DictIndex<'a>, count: usize, (i, batch): (usize, &Batch)) {
-    format_batch_aux(dict_idx, count, i, batch).expect("format batch");
-}
-
-fn format_batch_aux<'a>(
-    dict_idx: &DictIndex<'a>,
-    count: usize,
-    i: usize,
-    batch: &Batch,
-) -> Result<()> {
-    let mut f = io::BufWriter::new(fs::File::create(format!("public/{:03}.html", i))?);
-    write!(
-        f,
-        r#"<!DOCTYPE html>
-        <html>
-            <head>
-                <meta charset=\"UTF-8\" />
-                <title>Batch #{:03}</title>
-                <link rel="stylesheet" type="text/css" href="style.css" />
-            </head>
-            <body><div class="batch_page">"#,
-        i
-    )?;
-
-    writeln!(f, r#"<p><a href="index.html">index</a>"#)?;
-    for j in 0..count {
-        if j != i {
-            writeln!(f, r#" <a href="{:03}.html">{:03}</a>"#, j, j)?;
-        } else {
-            writeln!(f, " {:03}", j)?;
-        }
-    }
-    writeln!(f, r#"</p>"#)?;
-    writeln!(f, "<p>Level: {}</p>", batch.level)?;
-
-    write!(f, r#"<p class="ja">"#)?;
-    let mut ex_prev = Charset::default();
-    for ex in batch.examples.iter() {
-        let ex_chars = ex.chars.inter(&batch.chars);
-        for c in ex_chars.diff(&ex_prev).chars().iter() {
-            write!(
-                f,
-                r#"<a href="https://jisho.org/search/{}%20%23kanji">{}</a>"#,
-                c, c
-            )?;
-        }
-        ex_prev = ex_prev.union(&ex_chars);
-    }
-    writeln!(f, r#"</p>"#)?;
-
-    for ex in batch.examples.iter() {
-        writeln!(f, "<hr />")?;
-        write!(f, r#"<p class="ja">"#)?;
-        for c in ex.ja.chars() {
-            if batch.chars.contains(c) {
-                write!(f, r#"<span class="char_cur">{}</span>"#, c)?;
-            } else if batch.chars_p1.contains(c) {
-                write!(f, r#"<span class="char_p1">{}</span>"#, c)?;
-            } else if batch.chars_p2.contains(c) {
-                write!(f, r#"<span class="char_p2">{}</span>"#, c)?;
-            } else if batch.chars_bad.contains(c) {
-                write!(f, r#"<span class="char_bad">{}</span>"#, c)?;
-            } else {
-                write!(f, "{}", c)?;
-            }
-        }
-        writeln!(f, "</p>")?;
-        writeln!(f, r#"<p class="en">{}</p>"#, ex.en)?;
-
-        writeln!(f, r#"<details><summary>Explanation</summary>"#)?;
-        let mut expl_batch = Vec::new();
-        let mut expl_all = Vec::new();
-        for word in ex.expl.split(|c| c == ' ' || c == '~') {
-            let (keb, reb) = expl_clean_word(word);
-            let wchars = Charset::new(keb);
-            if !wchars.intersects(&ex.chars) {
-                continue;
-            }
-            if let Some(ents) = dict_idx.get(keb) {
-                for ent in ents.iter() {
-                    if let Some(s) = dict_str(keb, reb, ent) {
-                        if wchars.intersects(&batch.chars) {
-                            expl_batch.push(s);
-                        } else {
-                            expl_all.push(s);
-                        }
-                    }
-                }
-            }
-        }
-        for be in expl_batch {
-            writeln!(f, r#"<p>{}</p>"#, be)?;
-        }
-        writeln!(f, r#"<p class="chars">"#)?;
-        for c in ex.chars.inter(&batch.chars).chars().iter() {
-            writeln!(
-                f,
-                r#"<a href="https://jisho.org/search/{}%20%23kanji">{}</a>"#,
-                c, c
-            )?;
-        }
-        writeln!(f, r#"</p>"#)?;
-        for be in expl_all {
-            writeln!(f, r#"<p>{}</p>"#, be)?;
-        }
-        writeln!(f, r#"</details>"#)?;
-    }
-
-    writeln!(f, "<hr />")?;
-    format_vocab(
-        &mut f,
-        &batch
-            .extra_vocab
-            .iter()
-            .filter(|v| batch.level.contains(&v.level))
-            .collect::<Vec<_>>(),
-        "Extra vocabulary (this level)",
-    )?;
-    format_vocab(
-        &mut f,
-        &batch
-            .extra_vocab
-            .iter()
-            .filter(|v| !batch.level.contains(&v.level))
-            .collect::<Vec<_>>(),
-        "Extra vocabulary (previous levels)",
-    )?;
-
-    writeln!(
-        f,
-        r#"<details><summary>Extra examples (reading practice)</summary><table class="extratable">"#
-    )?;
-    for ex in batch.extra_examples.iter() {
-        let mut expl1 = Vec::new();
-        let mut expl2 = Vec::new();
-        for word in ex.expl.split(|c| c == ' ' || c == '~') {
-            let (keb, reb) = expl_clean_word(word);
-            let wchars = Charset::new(keb);
-            if !wchars.intersects(&ex.chars) {
-                continue;
-            }
-            if let Some(ents) = dict_idx.get(keb) {
-                for ent in ents.iter() {
-                    if let Some(s) = dict_str_short(keb, reb, ent) {
-                        if wchars.intersects(&batch.chars) {
-                            expl1.push(s);
-                        } else {
-                            expl2.push(s);
-                        }
-                    }
-                }
-            }
-        }
-        expl1.extend(expl2.into_iter());
-        let expl = expl1.join("<br />");
-        writeln!(
-            f,
-            r#"<tr><td><details><summary class="tab_large2 font_ja">&nbsp;&nbsp;{}&nbsp;&nbsp;</summary><div style="text-align: center">{}<br />{}</div></details></td></tr>"#,
-            ex.ja, ex.en, expl
-        )?;
-    }
-    writeln!(f, r#"</table></details>"#)?;
-
-    writeln!(f, "<hr />")?;
-    writeln!(f, "<p>＼(≧▽≦)／</p>")?;
-
-    write!(f, "<div></body></html>")?;
-    f.flush()?;
-    Ok(())
-}
-
-fn format_vocab(f: &mut impl Write, vocab: &[&JlptVocab], t: &str) -> Result<()> {
-    if !vocab.is_empty() {
-        writeln!(
-            f,
-            r#"<details><summary>{}</summary><table class="vocabtable">"#,
-            t
-        )?;
-        for v in vocab {
-            writeln!(
-                f,
-                r#"<tr><td>{}</td><td>&nbsp;&nbsp;<span class="tab_large font_ja">{}</span>&nbsp;&nbsp;</td><td>{}</td><td class="font_ja">{}</td></tr>"#,
-                v.level, v.kanji, v.en, v.kana
-            )?;
-        }
-        writeln!(f, "</table></details>")?;
-    }
-    Ok(())
-}
-
-fn expl_clean_word(w: &str) -> (&str, Option<&str>) {
-    let mut ret = w;
-    for delim in ['(', '{', '['] {
-        if let Some((s, _)) = ret.split_once(delim) {
-            ret = s;
-        }
-    }
-    let p = w
-        .split_once('(')
-        .and_then(|(_, r)| r.split_once(')'))
-        .map(|(p, _)| p);
-    (ret, p)
-}
-
-fn dict_str_short<'a>(
-    qkeb: &str,
-    qreb: Option<&str>,
-    ent: &roxmltree::Node<'a, 'a>,
-) -> Option<String> {
-    let r_ele = ent.children().find(|x| x.has_tag_name("r_ele")).unwrap();
-    let reb = r_ele.children().find(|x| x.has_tag_name("reb")).unwrap();
-    let reb = reb.text().unwrap().trim();
-
-    if qreb.map(|x| x != reb).unwrap_or(false) {
-        return None;
-    }
-
-    Some(format!(
-        r#"<span class="font_ja">{} 【{}】</span>"#,
-        qkeb, reb
-    ))
-}
-
-fn dict_str<'a>(qkeb: &str, qreb: Option<&str>, ent: &roxmltree::Node<'a, 'a>) -> Option<String> {
-    let mut ret = dict_str_short(qkeb, qreb, ent)?;
-
-    for sense in ent.children().filter(|x| x.has_tag_name("sense")) {
-        if let Some(s) = sense.children().find(|x| x.has_tag_name("gloss")) {
-            ret.extend(format!(" {};", s.text().unwrap().trim()).chars());
-        }
-    }
-
-    if ret.chars().rev().next() == Some(';') {
-        ret.pop();
-    }
-    Some(ret)
-}
-
-fn format_index(batches: &[Batch], kanji_levels: &[(String, String)]) -> Result<()> {
-    let mut f = io::BufWriter::new(fs::File::create("public/index.html")?);
-    write!(
-        f,
-        r#"<!DOCTYPE html>
-        <html>
-            <head>
-                <meta charset=\"UTF-8\" />
-                <title>List of batches</title>
-                <link rel="stylesheet" type="text/css" href="style.css" />
-            </head>
-            <body><div class="index_page">"#
-    )?;
-
-    writeln!(f, r#"<p><a href="about.html">About / How-to</a></p><hr />"#)?;
-
-    writeln!(f, "<table>")?;
-    writeln!(f, "<tr><th>Num</th><th>Level</th><th>Kanji</th><th>Examples</th><th>Lesson-1</th><th>Lesson-2</th><th>Ignore</th></tr>")?;
-    for (i, batch) in batches.iter().enumerate() {
-        writeln!(
-            f,
-            r#"<tr><td><a href="{:03}.html">{:03}</a></td><td>{}</td><td class="font_ja">{}</td><td>&nbsp;&nbsp;{}</td><td class="font_ja">{}</td><td class="font_ja">{}</td><td class="font_ja">{}</td></tr>"#,
-            i,
-            i,
-            batch.level,
-            batch.chars.to_string(),
-            batch.examples.len(),
-            batch.chars_p1.to_string(),
-            batch.chars_p2.to_string(),
-            batch.chars_bad.to_string()
-        )?;
-    }
-    writeln!(f, r#"</table>"#)?;
-
-    writeln!(f, "<hr />")?;
-
-    let all_chars = Charset::from_iter(
-        batches
-            .iter()
-            .map(|x| x.chars.chars().iter().copied())
-            .flatten(),
-    );
-    writeln!(f, "<table>")?;
-    writeln!(
-        f,
-        r#"<tr><th>Level</th><th>Count</th><th width="60%">Kanji</th><th>Missing kanji</th></tr>"#
-    )?;
-    for (lvl, chars) in kanji_levels.iter() {
-        if lvl == "N0+" || lvl.ends_with("-10") {
-            continue;
-        }
-        let chars = Charset::new(chars);
-        let missing = chars.diff(&all_chars);
-        writeln!(
-            f,
-            r#"<tr><td>{}</td><td>{}</td><td class="font_ja">{}</td><td><span class="font_ja">{}</span> ({})</td></tr>"#,
-            lvl,
-            chars.len(),
-            chars.to_string(),
-            missing.to_string(),
-            missing.len()
-        )?;
-    }
-    writeln!(f, "</table>")?;
-
-    write!(f, "</div></body></html>")?;
-    f.flush()?;
-    Ok(())
-}
-
-fn format_about() -> Result<()> {
-    let mut f = io::BufWriter::new(fs::File::create("public/about.html")?);
-    write!(
-        f,
-        r#"<!DOCTYPE html>
-        <html>
-            <head>
-                <meta charset=\"UTF-8\" />
-                <title>Datagengo README</title>
-                <link rel="stylesheet" type="text/css" href="style.css" />
-            </head>
-            <body>"#
-    )?;
-
-    writeln!(f, r#"<div class="about_page">"#)?;
-    writeln!(
-        f,
-        r#"<p><a href="index.html">Back to lessons</a></p><hr />"#
-    )?;
-
-    writeln!(
-        f,
-        "{}",
-        markdown::to_html(&fs::read_to_string("README.md")?)
-    )?;
-
-    writeln!(f, r#"</div></body></html>"#)?;
-
-    Ok(())
-}
-- 
cgit v1.2.3