#!/usr/bin/env python3 # /// script # requires-python = ">=3.10" # /// """Generate src/tables.rs from the Microsoft mapping tables in data/. The inputs are the canonical Microsoft code page tables published by the Unicode Consortium (https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/). Each output table is a full 256-entry ``[Option; 256]``: MS-DOS pages are not uniformly ASCII in the low half (CP864 maps 0x25 to ARABIC PERCENT SIGN) and several pages leave byte values undefined (CP857/864/869/874). Run from the crate root: ``uv run tools/gen_tables.py`` """ from __future__ import annotations import subprocess from collections.abc import Sequence from pathlib import Path CODE_PAGES: Sequence[int] = ( 437, 737, 775, 850, 852, 855, 857, 860, 861, 862, 863, 864, 865, 866, 869, 874, ) DATA_DIR = Path(__file__).resolve().parent.parent / "data" OUTPUT = Path(__file__).resolve().parent.parent / "src" / "tables.rs" HEADER = """\ //! Generated code page tables — do not edit by hand. //! //! Source: the Unicode Consortium's Microsoft mapping tables //! (`VENDORS/MICSFT/PC/CP*.TXT`), vendored in `data/` and regenerated with //! `tools/gen_tables.py`. Each table maps every byte value `0x00..=0xFF` to //! its Unicode scalar, or `None` where the code page leaves the byte //! undefined. """ class TableError(Exception): """A mapping table failed to parse or violated an invariant.""" def parse_table(path: Path) -> list[int | None]: """Parse one CP*.TXT into a 256-entry byte-to-scalar table. Args: path: The mapping table file. Returns: A list indexed by byte value; ``None`` marks an undefined byte. Raises: TableError: If the file is missing or non-ASCII, a byte is out of range or listed twice, a byte is never listed, or two bytes map to the same scalar (the encoder requires a bijection). """ try: # The lone stray byte in canonical data is a DOS EOF marker (0x1A), # which is still ASCII; anything else should fail loudly. text = path.read_text(encoding="ascii") except (OSError, UnicodeDecodeError) as error: raise TableError(f"{path.name}: unreadable mapping table") from error table: list[int | None] = [None] * 256 seen: set[int] = set() for raw_line in text.splitlines(): line = raw_line.strip() if not line.startswith("0x"): continue columns = line.split("\t") byte = int(columns[0], 16) if byte > 0xFF: raise TableError(f"{path.name}: byte {byte:#x} out of range") if byte in seen: raise TableError(f"{path.name}: byte {byte:#04x} listed twice") seen.add(byte) if len(columns) >= 2 and columns[1].strip().startswith("0x"): table[byte] = int(columns[1], 16) if len(seen) != 256: missing = sorted(set(range(256)) - seen) raise TableError(f"{path.name}: bytes never listed: {missing}") defined = [scalar for scalar in table if scalar is not None] if len(defined) != len(set(defined)): raise TableError(f"{path.name}: table is not a bijection") return table def rust_entry(scalar: int | None) -> str: """Render one table slot as a Rust ``Option`` literal.""" if scalar is None: return "None" ch = chr(scalar) if ch.isprintable() and ch not in ("'", "\\"): return f"Some('{ch}')" return f"Some('\\u{{{scalar:04X}}}')" def render_table(code_page: int, table: Sequence[int | None]) -> str: """Render one code page as a Rust constant, eight entries per line.""" lines = [ f"/// Code page {code_page}, byte value to Unicode scalar.", f"pub(crate) const CP{code_page}: [Option; 256] = [", ] for row_start in range(0, 256, 8): row = ", ".join(rust_entry(table[i]) for i in range(row_start, row_start + 8)) lines.append(f" {row}, // {row_start:#04x}-{row_start + 7:#04x}") lines.append("];\n") return "\n".join(lines) def main() -> None: """Regenerate src/tables.rs from every table in CODE_PAGES.""" rendered = [HEADER] for code_page in CODE_PAGES: table = parse_table(DATA_DIR / f"CP{code_page}.TXT") rendered.append(render_table(code_page, table)) OUTPUT.write_text("\n".join(rendered), encoding="utf-8") subprocess.run(["rustfmt", "--edition", "2021", str(OUTPUT)], check=True) print(f"wrote {OUTPUT} ({len(CODE_PAGES)} tables)") if __name__ == "__main__": main()