path: root/src/datafiles.rs

use std::collections::HashMap;
use std::fs;
use std::io::{self, BufRead};
use std::sync::Arc;

use anyhow::Result;
use serde::{Deserialize, Serialize};

use crate::charset::Charset;

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct Example {
    pub ja: String,
    pub en: String,
    pub expl: String,
    #[serde(default)]
    pub furigana: Option<String>,
    pub id: Option<String>,
    pub chars: Charset,
}

// =====================================================================
//                      PARSING DATA FILES
// =====================================================================

#[derive(Serialize, Deserialize)]
pub struct DictEntry {
    pub reb: String,
    pub ent_seq: u64,
    pub sense: Box<[String]>,
}

pub type DictIndex = HashMap<String, Vec<Arc<DictEntry>>>;
pub fn index_jmdict(dict: &roxmltree::Document) -> DictIndex {
    let dict = dict
        .root()
        .children()
        .find(|x| x.has_tag_name("JMdict"))
        .unwrap();

    let mut ret: DictIndex = HashMap::new();
    for ent in dict.children().filter(|x| x.has_tag_name("entry")) {
        let r_ele = ent.children().find(|x| x.has_tag_name("r_ele")).unwrap();
        let reb = r_ele.children().find(|x| x.has_tag_name("reb")).unwrap();
        let reb = reb.text().unwrap().trim().to_string();

        let ent_seq = ent.children().find(|x| x.has_tag_name("ent_seq")).unwrap();
        let ent_seq = ent_seq.text().unwrap().trim().parse().unwrap();

        let sense = ent
            .children()
            .filter(|x| x.has_tag_name("sense"))
            .filter_map(|sense| sense.children().find(|x| x.has_tag_name("gloss")))
            .map(|gloss| gloss.text().unwrap().trim().to_string())
            .collect::<Vec<_>>()
            .into_boxed_slice();
        let parsed_ent = Arc::new(DictEntry {
            reb,
            ent_seq,
            sense,
        });

        for r in ent.children().filter(|x| x.has_tag_name("k_ele")) {
            if let Some(keb) = r.children().find(|x| x.has_tag_name("keb")) {
                let txt = keb.text().unwrap().trim().to_string();
                ret.entry(txt).or_default().push(parsed_ent.clone());
            }
        }
    }

    ret
}

pub fn read_jmdict_idx() -> Result<DictIndex> {
    let file = fs::read("data/jmdict_idx.json")?;
    Ok(serde_json::from_slice::<DictIndex>(&file)?)
}

pub fn parse_kanjidic() -> Result<Vec<(String, Charset)>> {
    let n3_kanji = Charset::new(&fs::read_to_string("data/n3_kanji.txt")?.trim());

    let file = fs::read_to_string("data/kanjidic2.xml")?;
    let xml = roxmltree::Document::parse(&file)?;
    let kanjidic = xml.root().first_child().unwrap();
    assert!(kanjidic.has_tag_name("kanjidic2"));

    let mut levels = HashMap::new();

    for x in kanjidic.children() {
        if !x.has_tag_name("character") {
            continue;
        }
        let mut literal = None;
        let mut jlpt = None;
        let mut grade = None;
        for y in x.children() {
            if y.has_tag_name("literal") {
                literal = y.text();
            }
            if y.has_tag_name("misc") {
                for z in y.children() {
                    if z.has_tag_name("jlpt") {
                        jlpt = z.text().and_then(|x| str::parse::<i32>(x).ok());
                    }
                    if z.has_tag_name("grade") {
                        grade = z.text().and_then(|x| str::parse::<i32>(x).ok());
                    }
                }
            }
        }
        match grade {
            Some(i) if i <= 6 => grade = Some(7),
            _ => (),
        }
        if let Some(lit) = literal {
            assert_eq!(lit.chars().count(), 1);
            let jlpt = match jlpt {
                Some(4) => Some(5),
                Some(3) => Some(4),
                Some(2) if n3_kanji.contains(lit.chars().next().unwrap()) => Some(3),
                x => x,
            };
            levels
                .entry((jlpt, grade))
                .or_insert(String::new())
                .extend(lit.chars());
        }
    }

    let mut levels = levels.into_iter().collect::<Vec<_>>();
    levels.sort_by_key(|((j, g), _)| match (j, g) {
        (Some(j), Some(g)) => (10 - *j) * 20 + *g,
        (Some(j), None) => (10 - *j) * 20 + 15,
        (None, Some(g)) => 1000 + *g,
        (None, None) => 1015,
    });

    let mut ret = Vec::new();
    let mut pc = Charset::default();
    for ((j, g), chars) in levels.into_iter() {
        let name = match (j, g) {
            (Some(j), Some(7)) => format!("N{}a", j),
            (Some(j), Some(8)) => format!("N{}b", j),
            (Some(j), Some(g)) => format!("N{}-{}", j, g),
            (Some(j), None) => format!("N{}+", j),
            (None, Some(7)) => format!("N0a"),
            (None, Some(8)) => format!("N0b"),
            (None, Some(g)) => format!("N0-{}", g),
            (None, None) => format!("N0+"),
        };
        let chars = Charset::new(chars).diff(&pc);
        pc = pc.union(&chars);
        ret.push((name, chars));
    }

    Ok(ret)
}

pub fn read_kanji_levels() -> Result<Vec<(String, String)>> {
    Ok(fs::read_to_string("data/kanji_levels.txt")?
        .lines()
        .filter_map(|l| l.split_once(": "))
        .map(|(l, k)| (l.to_string(), k.to_string()))
        .collect::<Vec<_>>())
}

pub fn read_examples(all_kanji: &Charset) -> Result<Vec<Example>> {
    let file = fs::File::open("data/examples.utf")?;

    let mut ret = Vec::new();
    let mut a = "".to_string();

    for (i, line) in io::BufReader::new(file).lines().enumerate() {
        let line = line?;
        if line.starts_with("A:") {
            a = line;
        } else if line.starts_with("B:") {
            let s = a.strip_prefix("A: ");
            let t = line.strip_prefix("B: ");
            if let (Some(a), Some(b)) = (s, t) {
                if let Some((ja, eng)) = a.split_once("\t") {
                    if let Some((eng, id)) = eng.split_once("#") {
                        ret.push(Example {
                            ja: ja.to_string(),
                            en: eng.to_string(),
                            expl: b.to_string(),
                            id: Some(id.to_string()),
                            chars: Charset::new(ja).inter(all_kanji),
                            furigana: None,
                        });
                    } else {
                        ret.push(Example {
                            ja: ja.to_string(),
                            en: eng.to_string(),
                            expl: b.to_string(),
                            id: None,
                            chars: Charset::new(ja).inter(all_kanji),
                            furigana: None,
                        });
                    }
                }
            }
        }
        if i % 10000 == 0 {
            info!("read examples: {}/300 (x1000)", i / 1000);
        }
    }

    Ok(ret)
}

pub fn read_furigana_overrides() -> Result<HashMap<String, String>> {
    let file = fs::File::open("data/furigana_overrides")?;

    let mut ret = HashMap::new();
    let re = regex::Regex::new(r#"\|\|\w+\]\]"#)?;

    for line in io::BufReader::new(file).lines() {
        let line = line?;
        let line = line.trim();
        if !line.is_empty() {
            let clean = re.replace_all(line, "").replace("[[", "");
            if clean != line {
                ret.insert(clean, line.to_string());
            }
        }
    }

    Ok(ret)
}

#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct JlptVocab {
    pub level: String,
    pub chars: Charset,
    pub kanji: String,
    pub kana: String,
    pub en: String,
}

impl JlptVocab {
    pub fn to_string(&self) -> String {
        format!(
            "{}\t{}\t{}\t{}\t{}",
            self.level,
            self.chars.to_string(),
            self.kanji,
            self.kana,
            self.en
        )
    }
}

pub fn parse_jlpt_vocab(all_kanji: &Charset) -> Result<()> {
    let mut vocab = vec![];
    vocab.extend(parse_jlpt_vocab_combined(
        "data/n5_vocab.txt",
        "N5",
        all_kanji,
    )?);
    vocab.extend(parse_jlpt_vocab_split(
        "data/n4_vocab_hiragana.txt",
        "data/n4_vocab_eng.txt",
        "N4",
        all_kanji,
    )?);
    vocab.extend(parse_jlpt_vocab_split(
        "data/n3_vocab_hiragana.txt",
        "data/n3_vocab_eng.txt",
        "N3",
        all_kanji,
    )?);
    vocab.extend(parse_jlpt_vocab_split(
        "data/n2_vocab_hiragana.txt",
        "data/n2_vocab_eng.txt",
        "N2",
        all_kanji,
    )?);
    vocab.extend(parse_jlpt_vocab_split(
        "data/n1_vocab_hiragana.txt",
        "data/n1_vocab_eng.txt",
        "N1",
        all_kanji,
    )?);
    for v in vocab.iter() {
        println!("{}", v.to_string());
    }
    Ok(())
}

fn parse_jlpt_vocab_combined(
    file: &str,
    level: &str,
    all_kanji: &Charset,
) -> Result<Vec<JlptVocab>> {
    let lines = jlpt_vocab_read_file(file)?;
    let mut ret = vec![];
    for (kanji, answer) in lines {
        let (eng, kana) = match answer.split_once('\n') {
            Some((a, b)) => (a, b.trim()),
            None => (answer.trim(), ""),
        };
        for kanji in kanji.split('/') {
            ret.push(JlptVocab {
                level: level.to_string(),
                chars: Charset::new(kanji).inter(all_kanji),
                kanji: kanji.to_string(),
                kana: kana.to_string(),
                en: eng.to_string(),
            });
        }
    }
    Ok(ret)
}

fn parse_jlpt_vocab_split(
    kana_file: &str,
    eng_file: &str,
    level: &str,
    all_kanji: &Charset,
) -> Result<Vec<JlptVocab>> {
    let eng_lines = jlpt_vocab_read_file(eng_file)?
        .into_iter()
        .collect::<HashMap<String, String>>();

    let lines = jlpt_vocab_read_file(kana_file)?;
    let mut ret = vec![];
    for (kanji, kana) in lines {
        let eng = eng_lines.get(&kanji).or(eng_lines.get(&kana));
        if let Some(eng) = eng {
            for kanji in kanji.split('/') {
                ret.push(JlptVocab {
                    level: level.to_string(),
                    chars: Charset::new(kanji).inter(all_kanji),
                    kanji: kanji.to_string(),
                    kana: kana.to_string(),
                    en: eng.to_string(),
                });
            }
        }
    }
    Ok(ret)
}

fn jlpt_vocab_read_file(file: &str) -> Result<Vec<(String, String)>> {
    let re = regex::Regex::new(r#"<span class="\w+">"#)?;

    let file = fs::File::open(file)?;
    let mut ret = vec![];
    for line in io::BufReader::new(file).lines() {
        let line = line?.replace("<br>", "\n").replace("</span>", "");
        let line = re.replace_all(&line, "");
        if let Some((a, b)) = line.split_once('|') {
            ret.push((a.trim().to_string(), b.trim().to_string()));
        }
    }

    Ok(ret)
}

pub fn load_jlpt_vocab() -> Result<Vec<JlptVocab>> {
    let file = fs::File::open("data/jlpt_vocab.txt")?;
    let mut ret = vec![];
    for line in io::BufReader::new(file).lines() {
        let line = line?;
        let line = line.splitn(5, "\t").collect::<Vec<_>>();
        if line.len() == 5 {
            ret.push(JlptVocab {
                level: line[0].to_string(),
                chars: Charset::new(line[1]),
                kanji: line[2].to_string(),
                kana: line[3].to_string(),
                en: line[4].to_string(),
            });
        }
    }
    Ok(ret)
}