#!/usr/bin/env python3
# /// script
# requires-python = ">=3.10"
# ///
"""Generate src/tables/ from the Microsoft mapping tables in data/.

The inputs are the canonical Microsoft code page tables published by the
Unicode Consortium (https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/):
``PC/`` (vendored in ``data/pc/``) for the single-byte DOS pages and
``WINDOWS/`` (vendored in ``data/windows/``) for the Windows ANSI pages and
the double-byte CJK pages.

Single-byte pages become full 256-entry ``[Option<char>; 256]`` tables:
MS-DOS pages are not uniformly ASCII in the low half (CP864 maps 0x25 to
ARABIC PERCENT SIGN) and several pages leave byte values undefined
(CP857/864/869/874, most of the ANSI pages).

Double-byte pages become a 256-entry single-byte/lead-byte table plus a pair
of sorted ``(code, scalar)`` arrays for binary search. Where several codes
decode to one scalar (CP932's NEC/IBM overlap, CP950's box-drawing rows),
the encoder's choice is resolved from Microsoft's own ``bestfit*.txt``
WCTABLE — used *only* to pick among duplicate candidates, never to import
lossy best-fit mappings.

Run from the crate root: ``uv run tools/gen_tables.py``
(needs ``rustfmt`` on PATH).
"""

from __future__ import annotations

import re
import subprocess
from collections.abc import Sequence
from dataclasses import dataclass
from pathlib import Path

DOS_PAGES: Sequence[int] = (
    437, 737, 775, 850, 852, 855, 857, 860,
    861, 862, 863, 864, 865, 866, 869, 874,
)
ANSI_PAGES: Sequence[int] = (
    874, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258,
)
DOUBLE_BYTE_PAGES: Sequence[int] = (932, 936, 949, 950)

DATA_DIR = Path(__file__).resolve().parent.parent / "data"
PC_DIR = DATA_DIR / "pc"
WINDOWS_DIR = DATA_DIR / "windows"
BESTFIT_DIR = DATA_DIR / "bestfit"
OUTPUT_DIR = Path(__file__).resolve().parent.parent / "src" / "tables"

GENERATED_NOTE = """\
//! Generated by `tools/gen_tables.py` — do not edit by hand.
//!
//! Source: the Unicode Consortium's Microsoft mapping tables, vendored in
//! `data/`.
"""

SURROGATE_RANGE = range(0xD800, 0xE000)


class TableError(Exception):
    """A mapping table failed to parse or violated an invariant."""


@dataclass
class CodePageTable:
    """One parsed Microsoft mapping table.

    Attributes:
        single: Byte value to Unicode scalar for single-byte codes.
        leads: Byte values marked ``DBCS LEAD BYTE`` (empty for single-byte
            pages).
        pairs: 16-bit lead<<8|trail code to Unicode scalar.
        listed_bytes: Every byte value the file listed, mapped or not.
    """

    single: dict[int, int]
    leads: set[int]
    pairs: dict[int, int]
    listed_bytes: set[int]


def read_mapping_lines(path: Path) -> list[str]:
    """Read a mapping file and return its data lines (those starting 0x).

    Raises:
        TableError: If the file is missing or non-ASCII (the lone stray byte
            in canonical data is a DOS EOF marker, 0x1A, which is still ASCII).
    """
    try:
        text = path.read_text(encoding="ascii")
    except (OSError, UnicodeDecodeError) as error:
        raise TableError(f"{path.name}: unreadable mapping table") from error
    return [line.strip() for line in text.splitlines() if line.strip().startswith("0x")]


def parse_table(path: Path) -> CodePageTable:
    """Parse one CP*.TXT into single-byte, lead-byte, and pair mappings.

    Raises:
        TableError: If a code is listed twice, a single byte is out of range,
            a scalar is a surrogate or beyond the BMP, or byte 0x00 appears
            as a DBCS trail (which would break NUL-terminated decoding).
    """
    table = CodePageTable(single={}, leads=set(), pairs={}, listed_bytes=set())
    seen: set[int] = set()
    for line in read_mapping_lines(path):
        columns = line.split("\t")
        code = int(columns[0], 16)
        if code in seen:
            raise TableError(f"{path.name}: code {code:#06x} listed twice")
        seen.add(code)
        if code > 0xFFFF:
            raise TableError(f"{path.name}: code {code:#x} out of range")
        if code <= 0xFF:
            table.listed_bytes.add(code)
        mapped = len(columns) >= 2 and columns[1].strip().startswith("0x")
        if not mapped:
            if code <= 0xFF and "DBCS LEAD BYTE" in line:
                table.leads.add(code)
            continue
        scalar = int(columns[1], 16)
        if scalar in SURROGATE_RANGE or scalar > 0xFFFF:
            raise TableError(f"{path.name}: scalar {scalar:#06x} not a BMP char")
        if code <= 0xFF:
            table.single[code] = scalar
        else:
            if code & 0xFF == 0:
                raise TableError(f"{path.name}: NUL trail byte in {code:#06x}")
            table.pairs[code] = scalar
    unmarked_leads = {code >> 8 for code in table.pairs} - table.leads
    if unmarked_leads:
        raise TableError(
            f"{path.name}: pairs under bytes not marked DBCS LEAD BYTE: "
            f"{[hex(lead) for lead in sorted(unmarked_leads)]}"
        )
    return table


def parse_wctable(path: Path) -> dict[int, int]:
    """Parse the WCTABLE (Unicode to codepage) section of a bestfit file.

    Raises:
        TableError: If the file is missing or has no WCTABLE section.
    """
    try:
        # Comments embed raw codepage-encoded glyph bytes; latin-1 accepts
        # any byte and the data lines we match are pure ASCII.
        text = path.read_text(encoding="latin-1")
    except OSError as error:
        raise TableError(f"{path.name}: unreadable bestfit table") from error
    _, marker, rest = text.partition("WCTABLE")
    if not marker:
        raise TableError(f"{path.name}: no WCTABLE section")
    section = rest.split("ENDCODEPAGE")[0]
    entry = re.compile(r"^0x([0-9a-fA-F]{4})\s+0x([0-9a-fA-F]+)")
    table: dict[int, int] = {}
    for line in section.splitlines():
        if match := entry.match(line.strip()):
            table[int(match.group(1), 16)] = int(match.group(2), 16)
    return table


def check_bijection(path_name: str, table: CodePageTable) -> None:
    """Require every scalar to have exactly one code (single-byte pages).

    Raises:
        TableError: If two codes map to the same scalar.
    """
    scalars = list(table.single.values()) + list(table.pairs.values())
    if len(scalars) != len(set(scalars)):
        raise TableError(f"{path_name}: table is not a bijection")


def resolve_encode(
    path_name: str, table: CodePageTable, wctable: dict[int, int]
) -> dict[int, int]:
    """Choose one code per scalar, resolving duplicates via the WCTABLE.

    Args:
        path_name: The mapping file name, for error messages.
        table: The parsed decode-direction table.
        wctable: Microsoft's Unicode-to-codepage table for this page.

    Returns:
        Scalar to pair-code for the double-byte portion only (single-byte
        codes never collide with pairs in these tables, which is asserted).

    Raises:
        TableError: If a duplicate set is not settled by the WCTABLE, mixes
            single-byte and pair codes (the Rust encoder tries the single
            table first, so a pair winner would be unreachable), or has a
            single-byte winner that is not the lowest byte (the Rust encoder
            scans upward, so any other winner would be shadowed).
    """
    candidates: dict[int, list[int]] = {}
    for code, scalar in list(table.single.items()) + list(table.pairs.items()):
        candidates.setdefault(scalar, []).append(code)
    encode: dict[int, int] = {}
    for scalar, codes in candidates.items():
        if len(codes) == 1:
            chosen = codes[0]
        else:
            if any(c <= 0xFF for c in codes) and any(c > 0xFF for c in codes):
                raise TableError(
                    f"{path_name}: U+{scalar:04X} mixes single and pair codes "
                    f"({[hex(c) for c in codes]})"
                )
            chosen = wctable.get(scalar, -1)
            if chosen not in codes:
                raise TableError(
                    f"{path_name}: WCTABLE does not settle duplicate scalar "
                    f"U+{scalar:04X} (candidates {[hex(c) for c in codes]})"
                )
            if chosen <= 0xFF and chosen != min(codes):
                raise TableError(
                    f"{path_name}: U+{scalar:04X} single-byte winner "
                    f"{chosen:#04x} is not the lowest candidate"
                )
        if chosen <= 0xFF:
            continue  # single-byte encoding is derived from the single table
        encode[scalar] = chosen
    return encode


def rust_char(scalar: int) -> str:
    """Render a Unicode scalar as a Rust char literal."""
    ch = chr(scalar)
    if ch.isprintable() and ch not in ("'", "\\"):
        return f"'{ch}'"
    return f"'\\u{{{scalar:04X}}}'"


def render_single_byte_page(code_page: int, table: CodePageTable) -> str:
    """Render one single-byte code page as an ``[Option<char>; 256]``."""
    lines = [
        f"/// Code page {code_page}, byte value to Unicode scalar.",
        f"pub(crate) const CP{code_page}: [Option<char>; 256] = [",
    ]
    for row_start in range(0, 256, 8):
        entries = []
        for byte in range(row_start, row_start + 8):
            scalar = table.single.get(byte)
            entries.append("None" if scalar is None else f"Some({rust_char(scalar)})")
        lines.append(f"    {', '.join(entries)}, // {row_start:#04x}-{row_start + 7:#04x}")
    lines.append("];\n")
    return "\n".join(lines)


def render_pair_array(name: str, doc: str, pairs: Sequence[tuple[int, int]]) -> str:
    """Render a sorted ``[(u16, char); N]`` static, eight entries per line."""
    lines = [f"/// {doc}", f"static {name}: [(u16, char); {len(pairs)}] = ["]
    for row_start in range(0, len(pairs), 8):
        row = pairs[row_start : row_start + 8]
        rendered = ", ".join(f"({code:#06x}, {rust_char(scalar)})" for code, scalar in row)
        lines.append(f"    {rendered},")
    lines.append("];\n")
    return "\n".join(lines)


def render_encode_array(name: str, doc: str, pairs: Sequence[tuple[int, int]]) -> str:
    """Render a sorted ``[(char, u16); N]`` static, eight entries per line."""
    lines = [f"/// {doc}", f"static {name}: [(char, u16); {len(pairs)}] = ["]
    for row_start in range(0, len(pairs), 8):
        row = pairs[row_start : row_start + 8]
        rendered = ", ".join(f"({rust_char(scalar)}, {code:#06x})" for scalar, code in row)
        lines.append(f"    {rendered},")
    lines.append("];\n")
    return "\n".join(lines)


def render_double_byte_page(
    code_page: int, table: CodePageTable, encode: dict[int, int]
) -> str:
    """Render one DBCS page as a module body with single/decode/encode tables."""
    single_lines = [
        "/// Single-byte slots: mapped char, DBCS lead byte, or undefined.",
        "const SINGLE: [SingleEntry; 256] = [",
    ]
    for row_start in range(0, 256, 8):
        entries = []
        for byte in range(row_start, row_start + 8):
            if byte in table.leads:
                entries.append("L")
            elif (scalar := table.single.get(byte)) is not None:
                entries.append(f"M({rust_char(scalar)})")
            else:
                entries.append("U")
        single_lines.append(
            f"    {', '.join(entries)}, // {row_start:#04x}-{row_start + 7:#04x}"
        )
    single_lines.append("];\n")

    decode_pairs = sorted(table.pairs.items())
    encode_pairs = sorted(encode.items())
    return "\n".join(
        [
            GENERATED_NOTE,
            "use super::SingleEntry::{self, Lead as L, Map as M, Undefined as U};",
            "use super::DbcsPage;",
            "",
            f"/// Code page {code_page}.",
            f"pub(crate) static CP{code_page}: DbcsPage = DbcsPage {{",
            "    single: &SINGLE,",
            "    decode: &DECODE,",
            "    encode: &ENCODE,",
            "};",
            "",
            "\n".join(single_lines),
            render_pair_array(
                "DECODE",
                "Every defined pair, sorted by 16-bit code (lead<<8 | trail).",
                decode_pairs,
            ),
            render_encode_array(
                "ENCODE",
                "One pair per scalar, sorted by scalar; duplicates resolved per "
                "Microsoft's bestfit WCTABLE.",
                encode_pairs,
            ),
        ]
    )


def generate_single_byte_module(
    output_name: str, source_dir: Path, code_pages: Sequence[int]
) -> Path:
    """Render a family of single-byte pages into one tables module.

    Raises:
        TableError: If any page skips a byte value or contains DBCS content.
    """
    rendered = [GENERATED_NOTE]
    for code_page in code_pages:
        table = parse_table(source_dir / f"CP{code_page}.TXT")
        if table.listed_bytes != set(range(256)):
            missing = sorted(set(range(256)) - table.listed_bytes)
            raise TableError(f"CP{code_page}: bytes never listed: {missing}")
        if table.leads or table.pairs:
            raise TableError(f"CP{code_page}: unexpected DBCS content")
        check_bijection(f"CP{code_page}", table)
        rendered.append(render_single_byte_page(code_page, table))
    path = OUTPUT_DIR / output_name
    path.write_text("\n".join(rendered), encoding="utf-8")
    return path


def main() -> None:
    """Regenerate src/tables/ from every configured code page."""
    outputs = [
        generate_single_byte_module("dos.rs", PC_DIR, DOS_PAGES),
        generate_single_byte_module("ansi.rs", WINDOWS_DIR, ANSI_PAGES),
    ]

    for code_page in DOUBLE_BYTE_PAGES:
        table = parse_table(WINDOWS_DIR / f"CP{code_page}.TXT")
        wctable = parse_wctable(BESTFIT_DIR / f"bestfit{code_page}.txt")
        encode = resolve_encode(f"CP{code_page}", table, wctable)
        page_path = OUTPUT_DIR / f"cp{code_page}.rs"
        page_path.write_text(
            render_double_byte_page(code_page, table, encode), encoding="utf-8"
        )
        outputs.append(page_path)

    subprocess.run(
        ["rustfmt", "--edition", "2021", *map(str, outputs)], check=True
    )
    for path in outputs:
        print(f"wrote {path}")


if __name__ == "__main__":
    main()