path: root/src/main.rs

use std::collections::HashMap;
use std::fs;
use std::io::{self, BufRead, Write};

use rand::prelude::*;
use anyhow::{anyhow, Result};
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use structopt::StructOpt;

mod charset;
use charset::Charset;

#[derive(Debug, StructOpt)]
#[structopt(name = "datagengo", about = "Japanese example practice maker")]
struct Opt {
    #[structopt(subcommand)]
    cmd: Cmd,
}

#[derive(Debug, StructOpt)]
enum Cmd {
    ParseKanjidic,
    New {
        #[structopt(default_value = "10")]
        count: usize,
        #[structopt(long = "truncate")]
        truncate: Option<usize>,
    },
    Simplify,
    Rebalance {
        #[structopt(default_value = "5")]
        start: usize,
    },
    Format,
}

fn main() {
    let opt = Opt::from_args();

    match opt.cmd {
        Cmd::ParseKanjidic => {
            let levels = parse_kanjidic().expect("error");
            for (level, chars) in levels.iter() {
                println!("{}: {}", level, chars.to_string());
            }
        }
        Cmd::New { truncate, count } => {
            let kanji_levels = read_kanji_levels().expect("read_kanji_levels");
            let all_kanji = Charset::new(
                kanji_levels
                    .iter()
                    .map(|(_, x)| x.to_string())
                    .collect::<Vec<_>>()
                    .join(""),
            );
            let kanji_levels = kanji_levels
                .into_iter()
                .map(|(l, x)| (l, Charset::new(x)))
                .collect::<Vec<_>>();
            let mut ex = read_examples(&all_kanji).expect("read_examples");
            ex.retain(|e| (5..=25).contains(&e.ja.chars().count()));
            let mut batches: Vec<Batch> = fs::read("data/batches.json")
                .map_err(anyhow::Error::from)
                .and_then(|x| Ok(serde_json::from_slice(&x)?))
                .unwrap_or_default();
            if let Some(t) = truncate {
                batches.truncate(t);
            }
            println!("---- starting after {} batches ----", batches.len());
            let target_len = batches.len() + count;
            gen_batches(&mut batches, target_len, &kanji_levels, &ex);
            fs::write(
                "data/batches.json",
                serde_json::to_string_pretty(&batches)
                    .expect("serialize")
                    .as_bytes(),
            )
            .expect("save");
        }
        Cmd::Simplify => {
            let mut batches: Vec<Batch> = fs::read("data/batches.json")
                .map_err(anyhow::Error::from)
                .and_then(|x| Ok(serde_json::from_slice(&x)?))
                .unwrap_or_default();
            for batch in batches.iter_mut() {
                simplify_batch(batch);
            }
            fs::write(
                "data/batches.json",
                serde_json::to_string_pretty(&batches)
                    .expect("serialize")
                    .as_bytes(),
            )
            .expect("save");
        }
        Cmd::Rebalance { start } => {
            let mut batches: Vec<Batch> = fs::read("data/batches.json")
                .map_err(anyhow::Error::from)
                .and_then(|x| Ok(serde_json::from_slice(&x)?))
                .unwrap_or_default();
            let kanji_levels = read_kanji_levels().expect("read_kanji_levels");
            for (level, _) in kanji_levels.iter() {
                rebalance_level(level, &mut batches[start..]);
            }
            fs::write(
                "data/batches.json",
                serde_json::to_string_pretty(&batches)
                    .expect("serialize")
                    .as_bytes(),
            )
            .expect("save");
        }
        Cmd::Format => {
            let jmdict = fs::read_to_string("data/JMdict_e.xml").expect("read_jmdict");
            let jmdict = roxmltree::Document::parse_with_options(
                &jmdict,
                roxmltree::ParsingOptions {
                    allow_dtd: true,
                    ..Default::default()
                },
            )
            .expect("parse_jmdict");
            let jmdict_idx = index_jmdict(&jmdict);

            let batches = fs::read("data/batches.json")
                .map_err(anyhow::Error::from)
                .and_then(|x| Ok(serde_json::from_slice::<Vec<Batch>>(&x)?))
                .expect("read/parse");

            fs::create_dir_all("public").expect("mkdir public");
            fs::copy("static/style.css", "public/style.css").expect("copy style.css");

            batches
                .iter()
                .enumerate()
                .for_each(|x| format_batch(&jmdict_idx, batches.len(), x));

            let kanji_levels = read_kanji_levels().expect("read_kanji_levels");
            format_index(&batches, &kanji_levels).expect("format_index");
        }
    }
}

// =====================================================================
//                      PARSING DATA FILES
// =====================================================================

type DictIndex<'a> = HashMap<&'a str, Vec<roxmltree::Node<'a, 'a>>>;
fn index_jmdict<'a>(dict: &'a roxmltree::Document) -> DictIndex<'a> {
    let dict = dict
        .root()
        .children()
        .find(|x| x.has_tag_name("JMdict"))
        .unwrap();

    let mut ret: DictIndex<'a> = HashMap::new();
    for x in dict.children().filter(|x| x.has_tag_name("entry")) {
        for r in x.children().filter(|x| x.has_tag_name("k_ele")) {
            if let Some(keb) = r.children().find(|x| x.has_tag_name("keb")) {
                let txt = keb.text().unwrap().trim();
                ret.entry(txt).or_default().push(x);
            }
        }
    }

    ret
}

fn parse_kanjidic() -> Result<Vec<(String, Charset)>> {
    let file = fs::read_to_string("data/kanjidic2.xml")?;
    let xml = roxmltree::Document::parse(&file)?;
    let kanjidic = xml.root().first_child().unwrap();
    assert!(kanjidic.has_tag_name("kanjidic2"));

    let mut levels = HashMap::new();

    for x in kanjidic.children() {
        if !x.has_tag_name("character") {
            continue;
        }
        let mut literal = None;
        let mut jlpt = None;
        let mut grade = None;
        for y in x.children() {
            if y.has_tag_name("literal") {
                literal = y.text();
            }
            if y.has_tag_name("misc") {
                for z in y.children() {
                    if z.has_tag_name("jlpt") {
                        jlpt = z.text().and_then(|x| str::parse::<i32>(x).ok());
                    }
                    if z.has_tag_name("grade") {
                        grade = z.text().and_then(|x| str::parse::<i32>(x).ok());
                    }
                }
            }
        }
        match grade {
            Some(i) if i <= 6 => grade = Some(7),
            _ => (),
        }
        if let Some(lit) = literal {
            levels
                .entry((jlpt, grade))
                .or_insert(String::new())
                .extend(lit.chars());
        }
    }

    let mut levels = levels.into_iter().collect::<Vec<_>>();
    levels.sort_by_key(|((j, g), _)| match (j, g) {
        (Some(j), Some(g)) => (10 - *j) * 20 + *g,
        (Some(j), None) => (10 - *j) * 20 + 15,
        (None, Some(g)) => 1000 + *g,
        (None, None) => 1015,
    });

    let mut ret = Vec::new();
    let mut pc = Charset::default();
    for ((j, g), chars) in levels.into_iter() {
        let name = match (j, g) {
            (Some(j), Some(7)) => format!("N{}a", j),
            (Some(j), Some(8)) => format!("N{}b", j),
            (Some(j), Some(g)) => format!("N{}-{}", j, g),
            (Some(j), None) => format!("N{}+", j),
            (None, Some(7)) => format!("N0a"),
            (None, Some(8)) => format!("N0b"),
            (None, Some(g)) => format!("N0-{}", g),
            (None, None) => format!("N0+"),
        };
        let chars = Charset::new(chars).diff(&pc);
        pc = pc.union(&chars);
        ret.push((name, chars));
    }

    Ok(ret)
}

fn read_kanji_levels() -> Result<Vec<(String, String)>> {
    Ok(fs::read_to_string("data/kanji_levels.txt")?
        .lines()
        .filter_map(|l| l.split_once(": "))
        .map(|(l, k)| (l.to_string(), k.to_string()))
        .collect::<Vec<_>>())
}

fn read_examples(all_kanji: &Charset) -> Result<Vec<Example>> {
    let file = fs::File::open("data/examples.utf")?;

    let mut ret = Vec::new();
    let mut a = "".to_string();

    for (i, line) in io::BufReader::new(file).lines().enumerate() {
        let line = line?;
        if line.starts_with("A:") {
            a = line;
        } else if line.starts_with("B:") {
            let s = a.strip_prefix("A: ");
            let t = line.strip_prefix("B: ");
            if let (Some(a), Some(b)) = (s, t) {
                if let Some((ja, eng)) = a.split_once("\t") {
                    if let Some((eng, id)) = eng.split_once("#") {
                        ret.push(Example {
                            ja: ja.to_string(),
                            en: eng.to_string(),
                            expl: b.to_string(),
                            id: Some(id.to_string()),
                            chars: Charset::new(ja).inter(all_kanji),
                        });
                    } else {
                        ret.push(Example {
                            ja: ja.to_string(),
                            en: eng.to_string(),
                            expl: b.to_string(),
                            id: None,
                            chars: Charset::new(ja).inter(all_kanji),
                        });
                    }
                }
            }
        }
        if i % 10000 == 0 {
            eprintln!("read examples: {}/300 (x1000)", i / 1000);
        }
    }

    Ok(ret)
}

// =====================================================================
//                      BATCH STRUCTURES AND GENERATION
// =====================================================================

#[derive(Debug, Clone, Serialize, Deserialize)]
struct Example {
    ja: String,
    en: String,
    expl: String,
    id: Option<String>,
    chars: Charset,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
struct Batch {
    level: String,
    chars: Charset,
    chars_p1: Charset,
    chars_p2: Charset,
    chars_bad: Charset,
    examples: Vec<Example>,
}

fn gen_batches(
    batches: &mut Vec<Batch>,
    target_len: usize,
    kanji_levels: &[(String, Charset)],
    examples: &[Example],
) {
    let mut remainder = None;
    while batches.len() < target_len {
        let done = Charset::from_iter(
            batches
                .iter()
                .map(|x| x.chars.chars().iter().copied())
                .flatten()
                .chain(remainder.as_ref().map(|x: &Batch| x.chars.chars())
                       .unwrap_or_default().iter().copied()),
        );

        let mut advanced = false;
        for (i, (level, chars)) in kanji_levels.iter().enumerate() {
            let diff = chars.diff(&done);
            if !diff.is_empty() {
                let avoid = Charset::from_iter(
                    kanji_levels
                        .iter()
                        .skip(i + 1)
                        .filter(|(l, _)| !l.ends_with("-9") && !l.ends_with("-10"))
                        .map(|(_, c)| c.chars().iter().copied())
                        .flatten(),
                );

                let level_examples = level_examples(&diff, &avoid, examples);
                if !level_examples.is_empty() {
                    remainder = gen_level(batches, level, &diff, &done, &avoid, level_examples, remainder);
                    advanced = true;
                    break;
                }
            }
        }

        if !advanced {
            break;
        }
    }
    if let Some(r) = remainder {
        if batches.len() < target_len {
            batches.push(r);
        }
    }
}

fn gen_level(
    batches: &mut Vec<Batch>,
    level: &str,
    chars: &Charset,
    done: &Charset,
    avoid: &Charset,
    mut examples: Vec<&Example>,
    mut remainder: Option<Batch>,
) -> Option<Batch> {
    examples.shuffle(&mut thread_rng());

    println!(
        "Level {}: {} characters using {} examples",
        level,
        chars.len(),
        examples.len()
    );

    let avg_len = examples.len() as f32 * 20. / chars.len() as f32;

    let mut done = remainder.as_ref()
        .map(|x| x.chars.clone())
        .unwrap_or_default()
        .union(&done);

    'outer: while !examples.is_empty() {
        println!("iter with {} examples", examples.len());
        let mut batch = remainder.take().unwrap_or_else(|| Batch {
            level: level.to_string(),
            chars: Charset::default(),
            chars_p1: Charset::default(),
            chars_p2: Charset::default(),
            chars_bad: Charset::default(),
            examples: vec![],
        });
        let n_chars = 20 - batch.chars.len();

        let mut dyn_mat: Vec<Vec<Option<(Charset, Option<(usize, usize)>)>>> = vec![];
        for ex in examples.iter() {
            let mut dyn_row = vec![None; n_chars + 1];
            let chars_common = ex.chars.inter(&chars).diff(&done);
            if chars_common.len() > 6 {
                dyn_mat.push(dyn_row);
                continue;
            }
            if chars_common.len() < dyn_row.len() {
                dyn_row[chars_common.len()] = Some((chars_common.clone(), None));
            }
            for (i, dyn_prev) in dyn_mat.iter().enumerate() {
                for (j, dpr) in dyn_prev.iter().enumerate() {
                    if let Some((chars_inter, _prev)) = dpr {
                        assert_eq!(chars_inter.len(), j);
                        let new_chars_common = chars_inter.union(&chars_common);
                        let new_chars_common_len = new_chars_common.len();
                        if new_chars_common_len > chars_inter.len() && new_chars_common_len <= n_chars {
                            dyn_row[new_chars_common_len] = Some((new_chars_common, Some((i, j))));
                        }
                    }
                }
            }
            dyn_mat.push(dyn_row);
        }
        let dyn_mat_cnt = |i| {
            let mut i: usize = i;
            let mut cnt = 0;
            let mut j: usize = n_chars;
            loop {
                match &dyn_mat[i][j] {
                    None => return 1000000,
                    Some((_, ij_prev)) => {
                        cnt += 1;
                        match ij_prev {
                            Some((iprev, jprev)) => {
                                i = *iprev;
                                j = *jprev;
                            }
                            None => return cnt,
                        }
                    }
                }
            }
        };
        let i = (0..dyn_mat.len()).min_by_key(|i| {
            let x = dyn_mat_cnt(*i) as i64 - (avg_len as i64 + 2);
            x * x
        }).unwrap();
        let (mut i, mut j) = (i, n_chars);
        loop {
            match &dyn_mat[i][j] {
                None => break 'outer,
                Some((chars, ij_prev)) => {
                    println!("Add {}: {}", examples[i].chars.inter(&chars).to_string(), examples[i].ja);
                    batch.examples.push(examples[i].clone());
                    examples.remove(i);
                    batch.chars = batch.chars.union(&chars);
                    match ij_prev {
                        Some((iprev, jprev)) => {
                            assert!(*iprev < i);
                            i = *iprev;
                            j = *jprev;
                        }
                        None => break,
                    }
                }
            }
        }
        assert_eq!(batch.chars.len(), 20);

        batch.chars_p1 = batches
            .iter()
            .rev()
            .next()
            .map(|b| b.chars.clone())
            .unwrap_or(Charset::default())
            .inter(&batch.chars);

        batch.chars_p2 = batches
            .iter()
            .rev()
            .skip(1)
            .next()
            .map(|b| b.chars.clone())
            .unwrap_or(Charset::default())
            .inter(&batch.chars);

        batch.chars_bad = batch.chars.inter(&avoid);
        done = done.union(&batch.chars);

        println!("-> batch {:03}: {} with {} examples", batches.len(), batch.chars.to_string(), batch.examples.len());
        batches.push(batch);
    }

    if !examples.is_empty() {
        let n_remaining = chars.diff(&done).len();
        let ex_chars = Charset::from_iter(
            examples.iter()
            .map(|x| x.chars.chars().iter().copied())
            .flatten()).diff(&done);
        println!("still {} examples, {} chars in ex, {} total", examples.len(), ex_chars.len(), n_remaining);
        assert!(ex_chars.len() <= n_remaining);
        if ex_chars.len() > 20 {
            return None;
        }
        let remainder = Batch {
            level: level.to_string(),
            chars: ex_chars,
            chars_p1: Charset::default(),
            chars_p2: Charset::default(),
            chars_bad: Charset::default(),
            examples: examples.iter().map(|x| (**x).clone()).collect(),
        };
        println!("remainer batch: {:?}", remainder);
        Some(remainder)
    } else {
        None
    }
}

fn level_examples<'a>(
    chars: &Charset,
    avoid: &Charset,
    all_examples: &'a [Example],
) -> Vec<&'a Example> {
    println!("Calculating examples for {}", chars.to_string());

    let mut todo = chars.clone();
    let mut bad = Charset::default();
    let mut examples = vec![];

    let cost = |ex: &Example, ex_todo_inter: usize, ex_chars_inter: usize| {
        (
            -(ex.chars.inter_len(&avoid) as i32),
            ex_todo_inter,
            ex_chars_inter,
            -(ex.ja.chars().count() as i32),
            (thread_rng().gen::<usize>())
        )
    };
    let mut all_with_inter = all_examples
        .par_iter()
        .map(|ex| (ex, ex.chars.inter_len(&chars)))
        .map(|(ex, ex_chars_inter)| (ex, ex_chars_inter, ex_chars_inter))
        .collect::<Vec<_>>();

    while !todo.is_empty() {
        let best = all_with_inter.par_iter().enumerate()
            .filter(|(_, (_, ex_todo_inter, ex_tgt_inter))| (1..=6).contains(ex_todo_inter))
            .max_by_key(
            |(_, (ex, ex_todo_inter, ex_chars_inter))| cost(*ex, *ex_todo_inter, *ex_chars_inter),
        );
        if let Some((i, (ex, ex_todo_inter, _))) = best {
            let ex = *ex;
            assert_eq!(*ex_todo_inter, ex.chars.inter(&todo).len());
            examples.push(ex);
            all_with_inter.remove(i);
            todo = todo.diff(&ex.chars);
            bad = bad.union(&ex.chars.inter(&avoid));
            all_with_inter
                .par_iter_mut()
                .for_each(|(ex2, ex_todo_inter, _)| {
                    if ex2.chars.inter_len(&ex.chars) > 0 {
                        *ex_todo_inter = ex2.chars.inter_len(&todo);
                    }
                });
        } else {
            break;
        }
    }
    if !todo.is_empty() {
        println!("MISSING: NO SENTENCES FOR {}", todo.to_string());
    }
    if !bad.is_empty() {
        println!("USED BAD CHARS: {}", bad.to_string());
    }
    examples
}

/*

let (mut target_i, target_level, mut target_chars) = kanji_levels
    .iter()
    .enumerate()
    .map(|(i, (l, c))| (i, l, c.diff(&prev_chars)))
    .find(|(_, _, c)| !c.is_empty())
    .ok_or(anyhow!("no more batches to make!"))?;

let chars_p1 = previous
    .iter()
    .rev()
    .next()
    .map(|b| b.chars.clone())
    .unwrap_or(Charset::default());

let chars_p2 = previous
    .iter()
    .rev()
    .skip(1)
    .next()
    .map(|b| b.chars.clone())
    .unwrap_or(Charset::default());

let mut chars_late = Charset::default();

let mut chars_bad = Charset::from_iter(
    kanji_levels
        .iter()
        .skip(target_i + 1)
        .map(|(_, c)| c.chars().iter().copied())
        .flatten(),
);
let mut chars_bad_avoid = Charset::from_iter(
    kanji_levels
        .iter()
        .skip(target_i + 1)
        .filter(|(l, _)| !l.ends_with("-9") && !l.ends_with("-10"))
        .map(|(_, c)| c.chars().iter().copied())
        .flatten(),
);

let mut batch = Batch {
    level: target_level.to_string(),
    chars: Charset::default(),
    chars_p1: Charset::default(),
    chars_p2: Charset::default(),
    chars_bad: Charset::default(),
    examples: Vec::new(),
};
let mut batch_chars = Charset::default();

eprintln!("----");
eprintln!("Level  :  {}", batch.level);
eprintln!("Target :  {}", target_chars.to_string());
eprintln!("Prev1  :  {}", chars_p1.to_string());
eprintln!("Prev2  :  {}", chars_p2.to_string());
eprintln!("Bad    :  {} characters", chars_bad.len());

let batch_len = 20;
let mut stalled = false;
while batch.chars.len() < batch_len && !target_chars.is_empty() {
    let need = batch_len - batch.chars.len();
    let should_add = need > target_chars.len() && target_chars.len() <= 3;
    if target_i + 1 < kanji_levels.len() && (should_add || stalled) {
        // upgrade to next level
        target_i += 1;
        chars_late = chars_late.union(&target_chars);
        target_chars = target_chars.union(&kanji_levels[target_i].1.diff(&prev_chars));
        chars_bad = chars_bad.diff(&target_chars);
        chars_bad_avoid = chars_bad_avoid.diff(&target_chars);
        if batch.examples.is_empty() {
            batch.level = kanji_levels[target_i].0.to_string();
        } else {
            batch.level = format!("{} + {}", batch.level, kanji_levels[target_i].0);
        }
        eprintln!("Level  :  {}", batch.level);
        eprintln!("Target:  {}", target_chars.to_string());
        eprintln!("Late   :  {}", chars_late.to_string());
        eprintln!("Bad    :  {} characters", chars_bad.len());
        stalled = false;
    }
    /*  this one works well enough
    let cost = |ex: &Example, ex_tgt_inter: usize| {
        20i32 * ex_tgt_inter as i32
                    + 30i32 * ex.chars.inter_len(&chars_late) as i32
                    + 6i32 * ex.chars.inter_len(&batch.chars) as i32
                    + 4i32 * ex.chars.inter_len(&chars_p1) as i32
                    + 3i32 * ex.chars.inter_len(&chars_p2) as i32
                    - 40i32 * ex.chars.inter_len(&chars_bad) as i32
    };
    */
    let cost = |ex: &Example, ex_tgt_inter: usize| {
        (
            -(ex.chars.inter_len(&chars_bad_avoid) as i32),
            ex_tgt_inter,
            ex.chars.inter_len(&chars_late),
            2 * ex.chars.inter_len(&chars_p1) + ex.chars.inter_len(&chars_p2),
            -(ex.ja.chars().count() as i32),
        )
    };
    let cand_1 = examples
        .par_iter()
        .map(|ex| (ex, ex.chars.inter_len(&target_chars)))
        .filter(|(_, ex_tgt_inter)| {
            (1..=4).contains(ex_tgt_inter) && *ex_tgt_inter + batch.chars.len() <= batch_len
        })
        .max_by_key(|(ex, ex_tgt_inter)| cost(ex, *ex_tgt_inter));
    let cand = cand_1.or_else(|| {
        examples
            .par_iter()
            .map(|ex| (ex, ex.chars.inter_len(&target_chars)))
            .filter(|(_, ex_tgt_inter)| *ex_tgt_inter > 0)
            .max_by_key(|(ex, ex_tgt_inter)| cost(ex, *ex_tgt_inter))
    });
    if let Some((ex, _)) = cand {
        eprintln!(
            "* add {} (rep: {}, p1: {}, p2: {}, bad: {})    {}",
            ex.chars.inter(&target_chars).to_string(),
            ex.chars.inter(&batch.chars).to_string(),
            ex.chars.inter(&chars_p1).to_string(),
            ex.chars.inter(&chars_p2).to_string(),
            ex.chars.inter(&chars_bad).to_string(),
            ex.ja
        );
        batch.chars = batch.chars.union(&ex.chars.inter(&target_chars));
        target_chars = target_chars.diff(&ex.chars);
        chars_late = chars_late.diff(&ex.chars);
        batch.examples.push(ex.clone());
        batch_chars = batch_chars.union(&ex.chars);
        stalled = false;
    } else {
        if stalled {
            eprintln!(
                "could not find suitable sentence, stopping batch now (need {})",
                need
            );
            break;
        }
        stalled = true;
    }
}

batch.chars_p1 = chars_p1.inter(&batch_chars);
batch.chars_p2 = chars_p2.inter(&batch_chars);
batch.chars_bad = chars_bad.inter(&batch_chars);

Ok(batch)
    */

fn simplify_batch(batch: &mut Batch) {
    let mut char_cnt = HashMap::<char, usize>::new();
    for ex in batch.examples.iter() {
        for ch in batch.chars.inter(&ex.chars).chars() {
            *char_cnt.entry(*ch).or_default() += 1;
        }
    }

    loop {
        let i_opt = batch.examples.iter().position(|ex| {
            batch
                .chars
                .inter(&ex.chars)
                .chars()
                .iter()
                .all(|x| char_cnt[x] >= 2)
        });
        if let Some(i) = i_opt {
            println!(
                "Removing {} [{}]",
                batch.examples[i].ja,
                batch.examples[i].chars.to_string()
            );
            batch.examples.remove(i);
        } else {
            break;
        }
    }
}

fn rebalance_level(level: &str, batches: &mut [Batch]) {
    let mut i_batch = vec![];
    let mut n_ex = 0;
    for (i, b) in batches.iter().enumerate() {
        if b.level == level {
            i_batch.push(i);
            n_ex += b.examples.len();
        }
    }
    if i_batch.len() < 2 {
        return;
    }
    println!(
        "Level {}: {} batches, {} examples, avg {:.2}",
        level,
        i_batch.len(),
        n_ex,
        n_ex as f32 / i_batch.len() as f32
    );
    //todo!()
}

// =====================================================================
//                          FORMATTING TO HTML
// =====================================================================

fn format_batch<'a>(dict_idx: &DictIndex<'a>, count: usize, (i, batch): (usize, &Batch)) {
    format_batch_aux(dict_idx, count, i, batch).expect("format batch");
}

fn format_batch_aux<'a>(
    dict_idx: &DictIndex<'a>,
    count: usize,
    i: usize,
    batch: &Batch,
) -> Result<()> {
    let mut f = io::BufWriter::new(fs::File::create(format!("public/{:03}.html", i))?);
    write!(
        f,
        r#"<!DOCTYPE html>
        <html>
            <head>
                <meta charset=\"UTF-8\" />
                <title>Batch #{:03}</title>
                <link rel="stylesheet" type="text/css" href="style.css" />
            </head>
            <body>"#,
        i
    )?;

    writeln!(f, r#"<p><a href="index.html">index</a>"#)?;
    for j in 0..count {
        if j != i {
            writeln!(f, r#" <a href="{:03}.html">{:03}</a>"#, j, j)?;
        } else {
            writeln!(f, " {:03}", j)?;
        }
    }
    writeln!(f, r#"</p>"#)?;
    writeln!(f, "<p>Level: {}</p>", batch.level)?;

    writeln!(
        f,
        r#"<p class="ja chars">【{}】</p>"#,
        batch.chars.to_string()
    )?;

    for ex in batch.examples.iter() {
        writeln!(f, "<hr />")?;
        write!(f, r#"<p class="ja">"#)?;
        for c in ex.ja.chars() {
            if batch.chars.contains(c) {
                write!(f, r#"<span class="char_cur">{}</span>"#, c)?;
            } else if batch.chars_p1.contains(c) {
                write!(f, r#"<span class="char_p1">{}</span>"#, c)?;
            } else if batch.chars_p2.contains(c) {
                write!(f, r#"<span class="char_p2">{}</span>"#, c)?;
            } else if batch.chars_bad.contains(c) {
                write!(f, r#"<span class="char_bad">{}</span>"#, c)?;
            } else {
                write!(f, "{}", c)?;
            }
        }
        writeln!(f, "</p>")?;
        writeln!(f, r#"<p class="en">{}</p>"#, ex.en)?;

        writeln!(f, r#"<details><summary>Explanation</summary>"#)?;
        let mut expl_batch = Vec::new();
        let mut expl_all = Vec::new();
        for word in ex.expl.split(|c| c == ' ' || c == '~') {
            let (keb, reb) = expl_clean_word(word);
            let wchars = Charset::new(keb);
            if !wchars.intersects(&ex.chars) {
                continue;
            }
            if let Some(ents) = dict_idx.get(keb) {
                for ent in ents.iter() {
                    if let Some(s) = dict_str(keb, reb, ent) {
                        if wchars.intersects(&batch.chars) {
                            expl_batch.push(s);
                        } else {
                            expl_all.push(s);
                        }
                    }
                }
            }
        }
        for be in expl_batch {
            writeln!(f, r#"<p>{}</p>"#, be)?;
        }
        writeln!(f, r#"<p class="chars">"#)?;
        for c in ex.chars.inter(&batch.chars).chars().iter() {
            writeln!(
                f,
                r#"<a href="https://jisho.org/search/{}%20%23kanji">{}</a>"#,
                c, c
            )?;
        }
        writeln!(f, r#"</p>"#)?;
        for be in expl_all {
            writeln!(f, r#"<p>{}</p>"#, be)?;
        }
        writeln!(f, r#"</details>"#)?;
    }

    write!(f, "</body></html>")?;
    f.flush()?;
    Ok(())
}

fn expl_clean_word(w: &str) -> (&str, Option<&str>) {
    let mut ret = w;
    for delim in ['(', '{', '['] {
        if let Some((s, _)) = ret.split_once(delim) {
            ret = s;
        }
    }
    let p = w
        .split_once('(')
        .and_then(|(_, r)| r.split_once(')'))
        .map(|(p, _)| p);
    (ret, p)
}

fn dict_str<'a>(qkeb: &str, qreb: Option<&str>, ent: &roxmltree::Node<'a, 'a>) -> Option<String> {
    let r_ele = ent.children().find(|x| x.has_tag_name("r_ele")).unwrap();
    let reb = r_ele.children().find(|x| x.has_tag_name("reb")).unwrap();
    let reb = reb.text().unwrap().trim();

    if qreb.map(|x| x != reb).unwrap_or(false) {
        return None;
    }

    let mut ret = format!("{} [{}]", qkeb, reb);

    for sense in ent.children().filter(|x| x.has_tag_name("sense")) {
        if let Some(s) = sense.children().find(|x| x.has_tag_name("gloss")) {
            ret.extend(format!(" {};", s.text().unwrap().trim()).chars());
        }
    }

    if ret.chars().rev().next() == Some(';') {
        ret.pop();
    }
    Some(ret)
}

fn format_index(batches: &[Batch], kanji_levels: &[(String, String)]) -> Result<()> {
    let mut f = io::BufWriter::new(fs::File::create("public/index.html")?);
    write!(
        f,
        r#"<!DOCTYPE html>
        <html>
            <head>
                <meta charset=\"UTF-8\" />
                <title>List of batches</title>
                <link rel="stylesheet" type="text/css" href="style.css" />
            </head>
            <body>"#
    )?;

    writeln!(f, "<table>")?;
    writeln!(f, "<tr><th>Num</th><th>Level</th><th>Chars</th><th>Examples</th><th>Review</th><th>Ignore</th></tr>")?;
    for (i, batch) in batches.iter().enumerate() {
        writeln!(
            f,
            r#"<tr><td><a href="{:03}.html">{:03}</a></td><td>{}</td><td>{}</td><td>&nbsp;&nbsp;{}</td><td>{}</td><td>{}</td></tr>"#,
            i,
            i,
            batch.level,
            batch.chars.to_string(),
            batch.examples.len(),
            batch.chars_p1.union(&batch.chars_p2).to_string(),
            batch.chars_bad.to_string()
        )?;
    }
    writeln!(f, r#"</table>"#)?;

    writeln!(f, "<hr />")?;

    let all_chars = Charset::from_iter(
        batches
            .iter()
            .map(|x| x.chars.chars().iter().copied())
            .flatten(),
    );
    writeln!(f, "<table>")?;
    writeln!(
        f,
        "<tr><th>Level</th><th>Count</th><th>Chars</th><th>Missing chars</th></tr>"
    )?;
    for (lvl, chars) in kanji_levels.iter() {
        if lvl == "N0+" || lvl == "N0-9" || lvl.ends_with("-10") {
            continue;
        }
        let chars = Charset::new(chars);
        let missing = chars.diff(&all_chars);
        writeln!(
            f,
            "<tr><td>{}</td><td>{}</td><td>{}</td><td>{} ({})</td></tr>",
            lvl,
            chars.len(),
            chars.to_string(),
            missing.to_string(),
            missing.len()
        )?;
    }
    writeln!(f, "</table>")?;

    write!(f, "</body></html>")?;
    f.flush()?;
    Ok(())
}