msdos-encodings/tools/gen_tables.py

#!/usr/bin/env python3
# /// script
# requires-python = ">=3.10"
# ///
"""Generate src/tables.rs from the Microsoft mapping tables in data/.

The inputs are the canonical Microsoft code page tables published by the
Unicode Consortium (https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/).
Each output table is a full 256-entry ``[Option<char>; 256]``: MS-DOS pages are
not uniformly ASCII in the low half (CP864 maps 0x25 to ARABIC PERCENT SIGN)
and several pages leave byte values undefined (CP857/864/869/874).

Run from the crate root: ``uv run tools/gen_tables.py``
"""

from __future__ import annotations

import subprocess
from collections.abc import Sequence
from pathlib import Path

CODE_PAGES: Sequence[int] = (
    437, 737, 775, 850, 852, 855, 857, 860,
    861, 862, 863, 864, 865, 866, 869, 874,
)

DATA_DIR = Path(__file__).resolve().parent.parent / "data"
OUTPUT = Path(__file__).resolve().parent.parent / "src" / "tables.rs"

HEADER = """\
//! Generated code page tables — do not edit by hand.
//!
//! Source: the Unicode Consortium's Microsoft mapping tables
//! (`VENDORS/MICSFT/PC/CP*.TXT`), vendored in `data/` and regenerated with
//! `tools/gen_tables.py`. Each table maps every byte value `0x00..=0xFF` to
//! its Unicode scalar, or `None` where the code page leaves the byte
//! undefined.

"""


class TableError(Exception):
    """A mapping table failed to parse or violated an invariant."""


def parse_table(path: Path) -> list[int | None]:
    """Parse one CP*.TXT into a 256-entry byte-to-scalar table.

    Args:
        path: The mapping table file.

    Returns:
        A list indexed by byte value; ``None`` marks an undefined byte.

    Raises:
        TableError: If the file is missing or non-ASCII, a byte is out of
            range or listed twice, a byte is never listed, or two bytes map
            to the same scalar (the encoder requires a bijection).
    """
    try:
        # The lone stray byte in canonical data is a DOS EOF marker (0x1A),
        # which is still ASCII; anything else should fail loudly.
        text = path.read_text(encoding="ascii")
    except (OSError, UnicodeDecodeError) as error:
        raise TableError(f"{path.name}: unreadable mapping table") from error
    table: list[int | None] = [None] * 256
    seen: set[int] = set()
    for raw_line in text.splitlines():
        line = raw_line.strip()
        if not line.startswith("0x"):
            continue
        columns = line.split("\t")
        byte = int(columns[0], 16)
        if byte > 0xFF:
            raise TableError(f"{path.name}: byte {byte:#x} out of range")
        if byte in seen:
            raise TableError(f"{path.name}: byte {byte:#04x} listed twice")
        seen.add(byte)
        if len(columns) >= 2 and columns[1].strip().startswith("0x"):
            table[byte] = int(columns[1], 16)
    if len(seen) != 256:
        missing = sorted(set(range(256)) - seen)
        raise TableError(f"{path.name}: bytes never listed: {missing}")
    defined = [scalar for scalar in table if scalar is not None]
    if len(defined) != len(set(defined)):
        raise TableError(f"{path.name}: table is not a bijection")
    return table


def rust_entry(scalar: int | None) -> str:
    """Render one table slot as a Rust ``Option<char>`` literal."""
    if scalar is None:
        return "None"
    ch = chr(scalar)
    if ch.isprintable() and ch not in ("'", "\\"):
        return f"Some('{ch}')"
    return f"Some('\\u{{{scalar:04X}}}')"


def render_table(code_page: int, table: Sequence[int | None]) -> str:
    """Render one code page as a Rust constant, eight entries per line."""
    lines = [
        f"/// Code page {code_page}, byte value to Unicode scalar.",
        f"pub(crate) const CP{code_page}: [Option<char>; 256] = [",
    ]
    for row_start in range(0, 256, 8):
        row = ", ".join(rust_entry(table[i]) for i in range(row_start, row_start + 8))
        lines.append(f"    {row}, // {row_start:#04x}-{row_start + 7:#04x}")
    lines.append("];\n")
    return "\n".join(lines)


def main() -> None:
    """Regenerate src/tables.rs from every table in CODE_PAGES."""
    rendered = [HEADER]
    for code_page in CODE_PAGES:
        table = parse_table(DATA_DIR / f"CP{code_page}.TXT")
        rendered.append(render_table(code_page, table))
    OUTPUT.write_text("\n".join(rendered), encoding="utf-8")
    subprocess.run(["rustfmt", "--edition", "2021", str(OUTPUT)], check=True)
    print(f"wrote {OUTPUT} ({len(CODE_PAGES)} tables)")


if __name__ == "__main__":
    main()