apple-encodings/tools/gen_tables.py

#!/usr/bin/env python3
# /// script
# requires-python = ">=3.10"
# ///
"""Generate src/tables/mac.rs from the Apple mapping tables in data/apple/.

The inputs are the canonical Apple character set tables published by the
Unicode Consortium (https://www.unicode.org/Public/MAPPINGS/VENDORS/APPLE/),
vendored in ``data/apple/``. Each single-byte encoding becomes a full
256-entry ``[Option<char>; 256]`` table.

The vendored files carry the *modern* (post-Euro) mappings; the pre-Euro
revisions are reconstructed from the per-encoding deltas documented in each
file's change history (the Mac OS 8.5/9.0/9.2.2 Euro rollout landed at
different byte positions per encoding, which is why these are configured
data with invariant checks rather than hand-edited tables). Mac OS
Ukrainian, retired as a separate character set in Mac OS 9.0, is likewise
derived from CYRILLIC.TXT plus its documented delta (UKRAINE.TXT is a
notes-only stub).

Every delta records the scalar the vendored file is expected to hold, so
regeneration fails loudly if an upstream file ever changes underneath us.

Run from the crate root: ``uv run tools/gen_tables.py``
(needs ``rustfmt`` on PATH).
"""

from __future__ import annotations

import subprocess
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from pathlib import Path

DATA_DIR = Path(__file__).resolve().parent.parent / "data" / "apple"
OUTPUT_PATH = Path(__file__).resolve().parent.parent / "src" / "tables" / "mac.rs"

GENERATED_NOTE = """\
//! Generated by `tools/gen_tables.py` — do not edit by hand.
//!
//! Source: the Unicode Consortium's Apple mapping tables, vendored in
//! `data/apple/`. `*_CLASSIC` tables are the pre-Euro revisions,
//! reconstructed from the deltas documented in each file's change history.
"""

SURROGATE_RANGE = range(0xD800, 0xE000)

# Apple's files list 0x20-0x7E and 0x80-0xFF; the C0 controls and DELETE are
# omitted because they map to themselves.
CONTROL_BYTES = frozenset(range(0x20)) | {0x7F}
LISTED_BYTES = frozenset(range(0x100)) - CONTROL_BYTES

EURO = 0x20AC
CURRENCY_SIGN = 0x00A4
SOFT_HYPHEN = 0x00AD

# The Mac OS 8.5 Euro rollout for the Roman regional family: 0xDB flipped
# from CURRENCY SIGN to EURO SIGN.
ROMAN_FAMILY_EURO_DELTA: Mapping[int, tuple[int, int | None]] = {
    0xDB: (EURO, CURRENCY_SIGN),
}


class TableError(Exception):
    """A mapping table failed to parse or violated an invariant."""


@dataclass(frozen=True)
class EncodingSpec:
    """One generated encoding.

    Attributes:
        const: Rust constant name for the modern table.
        file: Mapping file name under ``data/apple/``.
        base_delta: Byte to ``(expected_modern_scalar, scalar_or_None)``
            applied to *derive* this encoding from the file (Mac OS
            Ukrainian is CYRILLIC.TXT with 0xFF back-flipped to CURRENCY
            SIGN). ``None`` marks the byte undefined.
        classic_delta: Byte to ``(expected_modern_scalar, scalar_or_None)``
            producing an additional ``*_CLASSIC`` (pre-Euro) table.
        doc: Override for the generated table's doc comment (used where the
            table is not simply the file's contents, e.g. Ukrainian).
    """

    const: str
    file: str
    base_delta: Mapping[int, tuple[int, int | None]] = field(default_factory=dict)
    classic_delta: Mapping[int, tuple[int, int | None]] = field(default_factory=dict)
    doc: str | None = None


SPECS: Sequence[EncodingSpec] = (
    EncodingSpec("MAC_ROMAN", "ROMAN.TXT", classic_delta=ROMAN_FAMILY_EURO_DELTA),
    # Mac OS 9.2.2 moved SOFT HYPHEN from 0x9C to the previously undefined
    # 0xFF and put EURO SIGN at 0x9C.
    EncodingSpec(
        "MAC_GREEK",
        "GREEK.TXT",
        classic_delta={0x9C: (EURO, SOFT_HYPHEN), 0xFF: (SOFT_HYPHEN, None)},
    ),
    # Mac OS 9.0 merged Cyrillic with Ukrainian (0xA2/0xB6 became GHE WITH
    # UPTURN) and put EURO SIGN at 0xFF.
    EncodingSpec(
        "MAC_CYRILLIC",
        "CYRILLIC.TXT",
        classic_delta={
            0xA2: (0x0490, 0x00A2),
            0xB6: (0x0491, 0x2202),
            0xFF: (EURO, CURRENCY_SIGN),
        },
    ),
    EncodingSpec("MAC_CENTRAL_EUR_ROMAN", "CENTEURO.TXT"),
    EncodingSpec("MAC_TURKISH", "TURKISH.TXT"),
    EncodingSpec("MAC_CROATIAN", "CROATIAN.TXT", classic_delta=ROMAN_FAMILY_EURO_DELTA),
    EncodingSpec("MAC_ICELANDIC", "ICELAND.TXT", classic_delta=ROMAN_FAMILY_EURO_DELTA),
    EncodingSpec("MAC_ROMANIAN", "ROMANIAN.TXT", classic_delta=ROMAN_FAMILY_EURO_DELTA),
    EncodingSpec("MAC_CELTIC", "CELTIC.TXT", classic_delta=ROMAN_FAMILY_EURO_DELTA),
    EncodingSpec("MAC_GAELIC", "GAELIC.TXT", classic_delta=ROMAN_FAMILY_EURO_DELTA),
    # The pre-9.0 Ukrainian currency sign variant: modern Cyrillic except
    # 0xFF stayed CURRENCY SIGN (per the notes in UKRAINE.TXT).
    EncodingSpec(
        "MAC_UKRAINIAN",
        "CYRILLIC.TXT",
        base_delta={0xFF: (EURO, CURRENCY_SIGN)},
        doc="CYRILLIC.TXT with 0xFF flipped back to CURRENCY SIGN — the "
        "pre-9.0 Ukrainian variant per the notes in UKRAINE.TXT.",
    ),
    EncodingSpec("MAC_INUIT", "INUIT.TXT"),
)


def parse_table(path: Path) -> dict[int, int]:
    """Parse one Apple mapping file into a full 256-entry byte-to-scalar map.

    The omitted C0 controls and DELETE are filled in as identity mappings.

    Raises:
        TableError: If the file is unreadable, lists a byte twice or outside
            the expected set, maps to anything but one BMP scalar, or is not
            ASCII-transparent over 0x20-0x7E.
    """
    try:
        text = path.read_text(encoding="ascii")
    except (OSError, UnicodeDecodeError) as error:
        raise TableError(f"{path.name}: unreadable mapping table") from error
    table = {byte: byte for byte in CONTROL_BYTES}
    seen: set[int] = set()
    for line in text.splitlines():
        line = line.strip()
        if not line.startswith("0x"):
            continue
        columns = line.split("\t")
        try:
            byte = int(columns[0], 16)
        except ValueError as error:
            raise TableError(f"{path.name}: malformed byte {columns[0]!r}") from error
        if byte in seen:
            raise TableError(f"{path.name}: byte {byte:#04x} listed twice")
        seen.add(byte)
        if byte not in LISTED_BYTES:
            raise TableError(f"{path.name}: unexpected byte {byte:#04x}")
        if len(columns) < 2 or not columns[1].startswith("0x"):
            raise TableError(f"{path.name}: byte {byte:#04x} has no mapping")
        scalar_text = columns[1]
        if "+" in scalar_text:
            raise TableError(f"{path.name}: byte {byte:#04x} maps to a sequence")
        try:
            scalar = int(scalar_text, 16)
        except ValueError as error:
            raise TableError(
                f"{path.name}: malformed scalar {scalar_text!r}"
            ) from error
        if scalar in SURROGATE_RANGE or scalar > 0xFFFF:
            raise TableError(f"{path.name}: scalar {scalar:#06x} not a BMP char")
        if 0x20 <= byte <= 0x7E and scalar != byte:
            raise TableError(f"{path.name}: byte {byte:#04x} is not ASCII")
        table[byte] = scalar
    if seen != LISTED_BYTES:
        missing = sorted(LISTED_BYTES - seen)
        raise TableError(f"{path.name}: bytes never listed: {missing}")
    return table


def apply_delta(
    name: str, table: Mapping[int, int], delta: Mapping[int, tuple[int, int | None]]
) -> dict[int, int | None]:
    """Apply a revision delta, verifying the modern scalars it replaces.

    Raises:
        TableError: If the vendored table does not hold the expected modern
            scalar at a delta byte — i.e. upstream data changed and the
            configured delta no longer describes it.
    """
    result: dict[int, int | None] = dict(table)
    for byte, (expected_modern, replacement) in delta.items():
        if table.get(byte) != expected_modern:
            raise TableError(
                f"{name}: delta expects U+{expected_modern:04X} at {byte:#04x}, "
                f"file has U+{table[byte]:04X}"
            )
        result[byte] = replacement
    return result


def check_bijection(name: str, table: Mapping[int, int | None]) -> None:
    """Require every mapped scalar to have exactly one byte.

    Raises:
        TableError: If two bytes map to the same scalar.
    """
    scalars = [scalar for scalar in table.values() if scalar is not None]
    if len(scalars) != len(set(scalars)):
        raise TableError(f"{name}: table is not a bijection")


def rust_char(scalar: int) -> str:
    """Render a Unicode scalar as a Rust char literal."""
    ch = chr(scalar)
    if ch.isprintable() and ch not in ("'", "\\"):
        return f"'{ch}'"
    return f"'\\u{{{scalar:04X}}}'"


def render_table(const: str, doc: str, table: Mapping[int, int | None]) -> str:
    """Render one encoding as an ``[Option<char>; 256]``."""
    lines = [f"/// {doc}", f"pub(crate) const {const}: [Option<char>; 256] = ["]
    for row_start in range(0, 256, 8):
        entries = []
        for byte in range(row_start, row_start + 8):
            scalar = table[byte]
            entries.append("None" if scalar is None else f"Some({rust_char(scalar)})")
        lines.append(
            f"    {', '.join(entries)}, // {row_start:#04x}-{row_start + 7:#04x}"
        )
    lines.append("];\n")
    return "\n".join(lines)


def render_spec(spec: EncodingSpec) -> list[str]:
    """Render a spec's modern table, plus its classic table if it has one.

    Raises:
        TableError: If parsing, a delta, or a bijection check fails.
    """
    file_table = parse_table(DATA_DIR / spec.file)
    modern = apply_delta(spec.const, file_table, spec.base_delta)
    check_bijection(spec.const, modern)
    doc = spec.doc or f"{spec.file}, byte value to Unicode scalar."
    rendered = [render_table(spec.const, doc, modern)]
    if spec.classic_delta:
        if spec.base_delta:
            raise TableError(f"{spec.const}: base and classic deltas both set")
        classic = apply_delta(spec.const, file_table, spec.classic_delta)
        check_bijection(f"{spec.const}_CLASSIC", classic)
        rendered.append(
            render_table(
                f"{spec.const}_CLASSIC",
                f"{spec.file} at the pre-Euro revision.",
                classic,
            )
        )
    return rendered


def main() -> None:
    """Regenerate src/tables/mac.rs from every configured encoding."""
    rendered = [GENERATED_NOTE]
    for spec in SPECS:
        rendered.extend(render_spec(spec))
    OUTPUT_PATH.write_text("\n".join(rendered), encoding="utf-8")
    subprocess.run(["rustfmt", "--edition", "2021", str(OUTPUT_PATH)], check=True)
    print(f"wrote {OUTPUT_PATH}")


if __name__ == "__main__":
    main()