write_text_sources.rs

//! Writes a page about text as a program receives it, before a page sets it.
//!
//! The first section writes one line as bytes seven ways — UTF-8 behind its
//! byte-order mark, UTF-16 with its mark, both orders of UTF-16 and of UTF-32,
//! and UTF-32 with no mark — and sets what `decode` reads back from each,
//! beside how many bytes it was and the first four of them. A lone half of a
//! UTF-16 surrogate pair is handed over last, and the page says it was refused.
//!
//! The second section holds one template kept in plain ASCII, with a named
//! reference, a numeric one, one by glyph name, a backslash escape and a
//! reference that names nothing, and sets what `resolve_references` makes of it
//! reading nothing, each kind alone, and all four.
//!
//! The third writes one word with its accent as a letter of its own after the
//! `e`, then as `to_nfc` composes it, each with its code points and whether
//! `is_nfc` finds it composed.
//!
//! The words are held in `Words`, once per language, and `HQF_PDF_LANG` picks
//! which set the page is written in. The template and the word stay as they are
//! in every language.
//!
//! Usage: `cargo run --example write_text_sources -- tmp/text_sources.pdf`
//!        `HQF_PDF_LANG=fr cargo run --example write_text_sources --
//! tmp/text_sources_fr.pdf`

use std::env;
use std::fs;
use std::path::{Path, PathBuf};

use hqf_pdf::content::Content;
use hqf_pdf::cos::Name;
use hqf_pdf::text::{References, TextFormat, decode, is_nfc, resolve_references, to_nfc};
use hqf_pdf::{Document, Font, FontHandle, Page};

#[path = "shared/out.rs"]
mod out;

#[path = "shared/licence.rs"]
mod licence;

#[path = "shared/language.rs"]
mod language;

#[path = "shared/failure.rs"]
mod failure;

use language::Language;

/// The font the page is written in.
fn default_font() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("tests")
        .join("fonts")
        .join("DejaVuSans.ttf")
}

/// How far in from the left edge of the sheet every line starts, in points.
const LEFT: f64 = 60.0;

/// Where the second column of a section starts, in points from the left edge.
const SECOND_COLUMN: f64 = 240.0;

/// Where the third column of the first section starts, in points from the left
/// edge.
const THIRD_COLUMN: f64 = 390.0;

/// Where the baseline of the title sits, in points up from the foot of the
/// sheet.
const TITLE_BASELINE: f64 = 780.0;

/// The size the title is set at, in points.
const TITLE_SIZE: f64 = 16.0;

/// The size a section's heading is set at, in points.
const HEADING_SIZE: f64 = 12.0;

/// The size every other line is set at, in points.
const BODY_SIZE: f64 = 9.5;

/// How far one line sits below the one before it, in points.
const LEADING: f64 = 16.0;

/// How far a heading sits below the line before it, in points.
const HEADING_GAP: f64 = 34.0;

/// The template of the second section: a named reference, a numeric one, one by
/// glyph name, a backslash escape and a reference that names nothing.
const TEMPLATE: &str = r"5 € — &:checkmark; \u{3A9} &nosuch;";

/// The word of the third section, its accent written after the `e` as a letter
/// of its own.
const APART: &str = "cafe\u{301}";

/// Every word the page says, in one language.
#[derive(Debug)]
struct Words {
    /// The line at the head of the page, and what the file says of itself.
    title: &'static str,
    /// The heading of the first section.
    bytes_heading: &'static str,
    /// The line the first section writes as bytes.
    sample: &'static str,
    /// What each way of writing it is called, in the order `formats` gives
    /// them.
    formats: [&'static str; 7],
    /// What follows a way's name; `{count}` and `{opening}` stand for how many
    /// bytes it took and the first four of them.
    bytes: &'static str,
    /// What the page says when the lone half of a surrogate pair is refused.
    refused: &'static str,
    /// What the page says if it is read all the same.
    accepted: &'static str,
    /// The heading of the second section.
    references_heading: &'static str,
    /// What stands before the template as it is written.
    written: &'static str,
    /// What each reading is called: nothing read, each kind alone, all four.
    readings: [&'static str; 6],
    /// The heading of the third section.
    nfc_heading: &'static str,
    /// What the word taken apart and the word composed are called.
    forms: [&'static str; 2],
    /// What the page says of a word `is_nfc` finds composed, and of one it does
    /// not.
    composed: [&'static str; 2],
}

impl Words {
    /// The words the page is written in, in `language`.
    fn of(language: Language) -> &'static Self {
        language::pick(&WORDS, language)
    }
}

/// The page in English.
const ENGLISH: Words = Words {
    title: "Text as it arrives, before a page sets it",
    bytes_heading: "One line written as bytes seven ways, and read back",
    sample: "Price: 5 € — paid ✓",
    formats: [
        "UTF-8 behind its mark",
        "UTF-16 with its mark",
        "UTF-16, big end first",
        "UTF-16, little end first",
        "UTF-32 with no mark",
        "UTF-32, big end first",
        "UTF-32, little end first",
    ],
    bytes: "{count} bytes, {opening}",
    refused: "Half a surrogate pair on its own, D8 00, is refused.",
    accepted: "Half a surrogate pair on its own, D8 00, is read.",
    references_heading: "One template in plain letters, read six ways",
    written: "As written",
    readings: [
        "Reading nothing",
        "Numeric references",
        "Named references",
        "Glyph names",
        "Backslash escapes",
        "All four",
    ],
    nfc_heading: "One accented letter, written two ways",
    forms: ["Taken apart", "Composed"],
    composed: ["composed", "not composed"],
};

/// The page in French.
const FRENCH: Words = Words {
    title: "Le texte tel qu’il arrive, avant qu’une page le compose",
    bytes_heading: "Une ligne écrite en octets de sept façons, et relue",
    sample: "Prix : 5 € — payé ✓",
    formats: [
        "UTF-8 précédé de sa marque",
        "UTF-16 avec sa marque",
        "UTF-16, poids fort d’abord",
        "UTF-16, poids faible d’abord",
        "UTF-32 sans marque",
        "UTF-32, poids fort d’abord",
        "UTF-32, poids faible d’abord",
    ],
    bytes: "{count} octets, {opening}",
    refused: "Une demi-paire de substitution isolée, D8 00, est refusée.",
    accepted: "Une demi-paire de substitution isolée, D8 00, est lue.",
    references_heading: "Un modèle en lettres simples, lu de six façons",
    written: "Tel qu’il est écrit",
    readings: [
        "Sans rien lire",
        "Références numériques",
        "Références nommées",
        "Noms de glyphes",
        "Échappements",
        "Les quatre",
    ],
    nfc_heading: "Une lettre accentuée, écrite de deux façons",
    forms: ["Décomposée", "Composée"],
    composed: ["composée", "non composée"],
};

/// The page in Dutch.
const DUTCH: Words = Words {
    title: "Tekst zoals hij binnenkomt, voordat een pagina hem zet",
    bytes_heading: "Eén regel op zeven manieren als bytes geschreven, en teruggelezen",
    sample: "Prijs: €\u{a0}5 — betaald ✓",
    formats: [
        "UTF-8 achter zijn markering",
        "UTF-16 met zijn markering",
        "UTF-16, grote kant eerst",
        "UTF-16, kleine kant eerst",
        "UTF-32 zonder markering",
        "UTF-32, grote kant eerst",
        "UTF-32, kleine kant eerst",
    ],
    bytes: "{count} bytes, {opening}",
    refused: "Een half surrogaatpaar op zichzelf, D8 00, wordt geweigerd.",
    accepted: "Een half surrogaatpaar op zichzelf, D8 00, wordt gelezen.",
    references_heading: "Eén sjabloon in gewone letters, op zes manieren gelezen",
    written: "Zoals geschreven",
    readings: [
        "Niets lezend",
        "Numerieke verwijzingen",
        "Benoemde verwijzingen",
        "Glyphnamen",
        "Escapes met backslash",
        "Alle vier",
    ],
    nfc_heading: "Eén letter met accent, op twee manieren geschreven",
    forms: ["Uit elkaar gehaald", "Samengesteld"],
    composed: ["samengesteld", "niet samengesteld"],
};

/// The page in Portuguese.
const PORTUGUESE: Words = Words {
    title: "Texto tal como chega, antes de uma página o compor",
    bytes_heading: "Uma linha escrita como bytes de sete formas, e relida",
    sample: "Preço: 5\u{a0}€ — pago ✓",
    formats: [
        "UTF-8 atrás da sua marca",
        "UTF-16 com a sua marca",
        "UTF-16, byte mais significativo primeiro",
        "UTF-16, byte menos significativo primeiro",
        "UTF-32 sem marca",
        "UTF-32, byte mais significativo primeiro",
        "UTF-32, byte menos significativo primeiro",
    ],
    bytes: "{count} bytes, {opening}",
    refused: "Meio par de substitutos sozinho, D8 00, é recusado.",
    accepted: "Meio par de substitutos sozinho, D8 00, é lido.",
    references_heading: "Um modelo em letras simples, lido de seis formas",
    written: "Tal como escrito",
    readings: [
        "Sem ler nada",
        "Referências numéricas",
        "Referências com nome",
        "Nomes de glifo",
        "Escapes com barra invertida",
        "Os quatro",
    ],
    nfc_heading: "Uma letra acentuada, escrita de duas formas",
    forms: ["Decomposta", "Composta"],
    composed: ["composta", "não composta"],
};

/// The page in Italian.
const ITALIAN: Words = Words {
    title: "Il testo come arriva, prima che una pagina lo componga",
    bytes_heading: "Una riga scritta come byte in sette modi, e riletta",
    sample: "Prezzo: 5\u{a0}€ — pagato ✓",
    formats: [
        "UTF-8 dietro il suo segno",
        "UTF-16 con il suo segno",
        "UTF-16, prima il byte più significativo",
        "UTF-16, prima il byte meno significativo",
        "UTF-32 senza segno",
        "UTF-32, prima il byte più significativo",
        "UTF-32, prima il byte meno significativo",
    ],
    bytes: "{count} byte, {opening}",
    refused: "Mezza coppia surrogata da sola, D8 00, è rifiutata.",
    accepted: "Mezza coppia surrogata da sola, D8 00, è letta.",
    references_heading: "Un modello in lettere semplici, letto in sei modi",
    written: "Come è scritto",
    readings: [
        "Senza leggere nulla",
        "Riferimenti numerici",
        "Riferimenti con nome",
        "Nomi di glifo",
        "Sequenze con barra rovesciata",
        "Tutti e quattro",
    ],
    nfc_heading: "Una lettera accentata, scritta in due modi",
    forms: ["Scomposta", "Composta"],
    composed: ["composta", "non composta"],
};

/// The page in Spanish.
const SPANISH: Words = Words {
    title: "Texto tal como llega, antes de que una página lo componga",
    bytes_heading: "Una línea escrita como bytes de siete formas, y leída de nuevo",
    sample: "Precio: 5\u{a0}€ — pagado ✓",
    formats: [
        "UTF-8 tras su marca",
        "UTF-16 con su marca",
        "UTF-16, byte más significativo primero",
        "UTF-16, byte menos significativo primero",
        "UTF-32 sin marca",
        "UTF-32, byte más significativo primero",
        "UTF-32, byte menos significativo primero",
    ],
    bytes: "{count} bytes, {opening}",
    refused: "Un sustituto sin pareja, D8 00, se rechaza.",
    accepted: "Un sustituto sin pareja, D8 00, se lee.",
    references_heading: "Una plantilla en letras simples, leída de seis formas",
    written: "Tal como se escribió",
    readings: [
        "Sin leer nada",
        "Referencias numéricas",
        "Referencias con nombre",
        "Nombres de glifo",
        "Escapes con barra invertida",
        "Las cuatro",
    ],
    nfc_heading: "Una letra acentuada, escrita de dos formas",
    forms: ["Descompuesta", "Compuesta"],
    composed: ["compuesta", "no compuesta"],
};

/// The page in German.
const GERMAN: Words = Words {
    title: "Text, wie er ankommt, bevor eine Seite ihn setzt",
    bytes_heading: "Eine Zeile, auf sieben Arten als Bytes geschrieben und zurückgelesen",
    sample: "Preis: 5 € — bezahlt ✓",
    formats: [
        "UTF-8 mit vorangestellter BOM",
        "UTF-16 mit BOM",
        "UTF-16, Big Endian",
        "UTF-16, Little Endian",
        "UTF-32 ohne BOM",
        "UTF-32, Big Endian",
        "UTF-32, Little Endian",
    ],
    bytes: "{count} Bytes, {opening}",
    refused: "Ein halbes Surrogatpaar für sich, D8 00, wird abgelehnt.",
    accepted: "Ein halbes Surrogatpaar für sich, D8 00, wird gelesen.",
    references_heading: "Eine Vorlage in einfachen Buchstaben, auf sechs Arten gelesen",
    written: "Wie geschrieben",
    readings: [
        "Nichts auflösen",
        "Numerische Referenzen",
        "Benannte Referenzen",
        "Glyphennamen",
        "Backslash-Maskierungen",
        "Alle vier",
    ],
    nfc_heading: "Ein Buchstabe mit Akzent, auf zwei Arten geschrieben",
    forms: ["Zerlegt", "Zusammengesetzt"],
    composed: ["zusammengesetzt", "nicht zusammengesetzt"],
};

/// Every language the example is written in. A language is added by writing its
/// own set of words and naming it here.
static WORDS: [(Language, &Words); 7] = [
    (Language::English, &ENGLISH),
    (Language::French, &FRENCH),
    (Language::German, &GERMAN),
    (Language::Spanish, &SPANISH),
    (Language::Italian, &ITALIAN),
    (Language::Portuguese, &PORTUGUESE),
    (Language::Dutch, &DUTCH),
];

/// `text` as UTF-16 units, each written in the order `big` says.
fn utf16(text: &str, big: bool) -> Vec<u8> {
    text.encode_utf16()
        .flat_map(|unit| {
            if big {
                unit.to_be_bytes()
            } else {
                unit.to_le_bytes()
            }
        })
        .collect()
}

/// `text` as UTF-32 units, each written in the order `big` says.
fn utf32(text: &str, big: bool) -> Vec<u8> {
    text.chars()
        .flat_map(|character| {
            let unit = u32::from(character);
            if big {
                unit.to_be_bytes()
            } else {
                unit.to_le_bytes()
            }
        })
        .collect()
}

/// `text` written as bytes each way the first section shows, with the form each
/// is read back in.
fn formats(text: &str) -> [(TextFormat, Vec<u8>); 7] {
    [
        (
            TextFormat::Utf8,
            [&[0xEF, 0xBB, 0xBF], text.as_bytes()].concat(),
        ),
        (
            TextFormat::Utf16,
            [vec![0xFF, 0xFE], utf16(text, false)].concat(),
        ),
        (TextFormat::Utf16Be, utf16(text, true)),
        (TextFormat::Utf16Le, utf16(text, false)),
        (TextFormat::Utf32, utf32(text, true)),
        (TextFormat::Utf32Be, utf32(text, true)),
        (TextFormat::Utf32Le, utf32(text, false)),
    ]
}

/// The first four of `bytes`, in hexadecimal, and an ellipsis.
fn opening(bytes: &[u8]) -> String {
    let shown: Vec<String> = bytes
        .iter()
        .take(4)
        .map(|byte| format!("{byte:02X}"))
        .collect();
    format!("{} …", shown.join(" "))
}

/// The ways the second section reads the template, in the order `readings`
/// names them.
const fn readings() -> [References; 6] {
    let none = References::none();
    [
        none,
        none.numeric(true),
        none.named(true),
        none.glyph_names(true),
        none.escapes(true),
        References::all(),
    ]
}

/// The code points of `text`, as `U+` and four hexadecimal digits each.
fn code_points(text: &str) -> String {
    let codes: Vec<String> = text
        .chars()
        .map(|character| format!("U+{:04X}", u32::from(character)))
        .collect();
    codes.join(" ")
}

/// Every line the page sets, in order: what it says, the size it is set at, how
/// far from the left edge it starts, and where its baseline sits.
fn lines(words: &Words) -> Result<Vec<(String, f64, f64, f64)>, hqf_pdf::Error> {
    let mut set = vec![(words.title.to_owned(), TITLE_SIZE, LEFT, TITLE_BASELINE)];
    let mut baseline = TITLE_BASELINE - HEADING_GAP;

    set.push((words.bytes_heading.to_owned(), HEADING_SIZE, LEFT, baseline));
    for ((format, bytes), name) in formats(words.sample).into_iter().zip(words.formats) {
        baseline -= LEADING;
        let said = words
            .bytes
            .replace("{count}", &bytes.len().to_string())
            .replace("{opening}", &opening(&bytes));
        set.push((name.to_owned(), BODY_SIZE, LEFT, baseline));
        set.push((said, BODY_SIZE, SECOND_COLUMN, baseline));
        set.push((decode(&bytes, format)?, BODY_SIZE, THIRD_COLUMN, baseline));
    }
    baseline -= LEADING;
    let refused = decode(&[0xD8, 0x00], TextFormat::Utf16Be).is_err();
    let verdict = if refused {
        words.refused
    } else {
        words.accepted
    };
    set.push((verdict.to_owned(), BODY_SIZE, LEFT, baseline));

    baseline -= HEADING_GAP;
    set.push((
        words.references_heading.to_owned(),
        HEADING_SIZE,
        LEFT,
        baseline,
    ));
    baseline -= LEADING;
    set.push((words.written.to_owned(), BODY_SIZE, LEFT, baseline));
    set.push((TEMPLATE.to_owned(), BODY_SIZE, SECOND_COLUMN, baseline));
    for (kinds, name) in readings().into_iter().zip(words.readings) {
        baseline -= LEADING;
        set.push((name.to_owned(), BODY_SIZE, LEFT, baseline));
        set.push((
            resolve_references(TEMPLATE, kinds).into_owned(),
            BODY_SIZE,
            SECOND_COLUMN,
            baseline,
        ));
    }

    baseline -= HEADING_GAP;
    set.push((words.nfc_heading.to_owned(), HEADING_SIZE, LEFT, baseline));
    let composed = to_nfc(APART).into_owned();
    for (word, form) in [APART.to_owned(), composed].into_iter().zip(words.forms) {
        baseline -= LEADING;
        let state = if is_nfc(&word) {
            words.composed[0]
        } else {
            words.composed[1]
        };
        set.push((format!("{form}, {state}"), BODY_SIZE, LEFT, baseline));
        set.push((code_points(&word), BODY_SIZE, SECOND_COLUMN, baseline));
    }
    Ok(set)
}

/// The page: every line set by its own origin, each in a text object of its
/// own.
fn page(font: &FontHandle, words: &Words) -> Result<Page, hqf_pdf::Error> {
    let mut content = Content::new();
    for (text, size, x, y) in lines(words)? {
        content.begin_text();
        content.set_font(font, size)?;
        content.text_origin(x, y)?;
        content.show_glyphs(&font.glyphs(&text));
        content.end_text();
    }
    let mut page = Page::a4();
    page.content = content.into_bytes();
    Ok(page)
}

fn main() -> std::process::ExitCode {
    failure::reported(run())
}

fn run() -> Result<(), Box<dyn std::error::Error>> {
    let language = Language::from_environment()?;
    let words = Words::of(language);

    // A named file is written as named; the default one carries the language,
    // so the two languages do not overwrite each other in `tmp/`.
    let path = env::args()
        .nth(1)
        .unwrap_or_else(|| language.file_name(&out::default_path("text_sources")));

    let mut doc = Document::new();
    doc.set_license(licence::licensed());
    doc.set_info(Name::new("Title"), words.title);
    let font = doc.add_font(Font::parse(fs::read(default_font())?)?);
    doc.add_page(page(&font, words)?)?;

    let bytes = doc.to_bytes()?;
    if let Some(parent) = Path::new(&path).parent() {
        fs::create_dir_all(parent)?;
    }
    fs::write(&path, &bytes)?;
    println!("wrote {} ({} bytes)", path, bytes.len());
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::{
        APART, ENGLISH, FRENCH, LEFT, TEMPLATE, WORDS, default_font, formats, language, lines,
        to_nfc,
    };
    use hqf_pdf::text::{References, decode, resolve_references};

    /// The lines two languages are allowed to write the same way, where two
    /// languages spell a word alike. Every line is words a person reads.
    const SPARED: [&str; 4] = [
        // Written the same way in Spanish.
        r#"bytes: "{count} bytes, {opening}""#,
        // Written the same way in Portuguese.
        r#""Composta""#,
        r#""composta""#,
        // Written the same way in Dutch.
        r#""Alle vier""#,
    ];

    #[test]
    fn every_language_writes_the_page_in_its_own_words() {
        let untranslated = language::untranslated_lines(&WORDS, &SPARED);

        assert!(
            untranslated.is_empty(),
            "the page says these in more than one language: {untranslated:?}"
        );
    }

    #[test]
    fn every_way_of_writing_the_line_reads_back_as_it() {
        for words in [&ENGLISH, &FRENCH] {
            for (format, bytes) in formats(words.sample) {
                assert_eq!(decode(&bytes, format).unwrap(), words.sample);
            }
        }
    }

    #[test]
    fn the_template_reads_into_its_characters_and_keeps_what_names_nothing() {
        assert_eq!(
            resolve_references(TEMPLATE, References::all()),
            "5 € — ✓ Ω &nosuch;"
        );
        assert_eq!(resolve_references(TEMPLATE, References::none()), TEMPLATE);
    }

    #[test]
    fn the_word_composes_into_one_letter_fewer() {
        assert_eq!(to_nfc(APART), "café");
    }

    /// Every line ends before the next one on its baseline starts, with a gap
    /// of eight points, or before the right margin, as wide as the left.
    #[test]
    fn every_line_ends_before_what_follows_it() {
        let mut doc = hqf_pdf::Document::new();
        let font = doc.add_font(
            hqf_pdf::Font::parse(
                std::fs::read(default_font()).expect("the committed font is there"),
            )
            .expect("the committed font parses"),
        );
        for words in [&ENGLISH, &FRENCH] {
            let set = lines(words).unwrap();
            for (text, size, x, y) in &set {
                let limit = set
                    .iter()
                    .filter(|(_, _, other, baseline)| (baseline - y).abs() < 0.5 && other > x)
                    .map(|(_, _, other, _)| other - 8.0)
                    .fold(hqf_pdf::Page::a4().width - LEFT, f64::min);
                let end = x + font.measure(text, *size);
                assert!(end <= limit, "{text:?} ends at {end:.1}, past {limit:.1}");
            }
        }
    }

    #[test]
    fn every_line_is_set() {
        for words in [&ENGLISH, &FRENCH] {
            assert_eq!(lines(words).unwrap().len(), 44);
        }
    }
}