948135af72
Tables are generated from the Unicode Consortium's Microsoft mappings (VENDORS/MICSFT/PC), vendored in data/ and regenerated by tools/gen_tables.py. Decode is fallible because 857/864/869/874 leave byte values undefined; decode_lossy substitutes U+FFFD. Every table is a verified bijection, so defined bytes round-trip exactly.
126 lines
4.4 KiB
Python
126 lines
4.4 KiB
Python
#!/usr/bin/env python3
|
|
# /// script
|
|
# requires-python = ">=3.10"
|
|
# ///
|
|
"""Generate src/tables.rs from the Microsoft mapping tables in data/.
|
|
|
|
The inputs are the canonical Microsoft code page tables published by the
|
|
Unicode Consortium (https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/).
|
|
Each output table is a full 256-entry ``[Option<char>; 256]``: MS-DOS pages are
|
|
not uniformly ASCII in the low half (CP864 maps 0x25 to ARABIC PERCENT SIGN)
|
|
and several pages leave byte values undefined (CP857/864/869/874).
|
|
|
|
Run from the crate root: ``uv run tools/gen_tables.py``
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
from collections.abc import Sequence
|
|
from pathlib import Path
|
|
|
|
CODE_PAGES: Sequence[int] = (
|
|
437, 737, 775, 850, 852, 855, 857, 860,
|
|
861, 862, 863, 864, 865, 866, 869, 874,
|
|
)
|
|
|
|
DATA_DIR = Path(__file__).resolve().parent.parent / "data"
|
|
OUTPUT = Path(__file__).resolve().parent.parent / "src" / "tables.rs"
|
|
|
|
HEADER = """\
|
|
//! Generated code page tables — do not edit by hand.
|
|
//!
|
|
//! Source: the Unicode Consortium's Microsoft mapping tables
|
|
//! (`VENDORS/MICSFT/PC/CP*.TXT`), vendored in `data/` and regenerated with
|
|
//! `tools/gen_tables.py`. Each table maps every byte value `0x00..=0xFF` to
|
|
//! its Unicode scalar, or `None` where the code page leaves the byte
|
|
//! undefined.
|
|
|
|
"""
|
|
|
|
|
|
class TableError(Exception):
|
|
"""A mapping table failed to parse or violated an invariant."""
|
|
|
|
|
|
def parse_table(path: Path) -> list[int | None]:
|
|
"""Parse one CP*.TXT into a 256-entry byte-to-scalar table.
|
|
|
|
Args:
|
|
path: The mapping table file.
|
|
|
|
Returns:
|
|
A list indexed by byte value; ``None`` marks an undefined byte.
|
|
|
|
Raises:
|
|
TableError: If the file is missing or non-ASCII, a byte is out of
|
|
range or listed twice, a byte is never listed, or two bytes map
|
|
to the same scalar (the encoder requires a bijection).
|
|
"""
|
|
try:
|
|
# The lone stray byte in canonical data is a DOS EOF marker (0x1A),
|
|
# which is still ASCII; anything else should fail loudly.
|
|
text = path.read_text(encoding="ascii")
|
|
except (OSError, UnicodeDecodeError) as error:
|
|
raise TableError(f"{path.name}: unreadable mapping table") from error
|
|
table: list[int | None] = [None] * 256
|
|
seen: set[int] = set()
|
|
for raw_line in text.splitlines():
|
|
line = raw_line.strip()
|
|
if not line.startswith("0x"):
|
|
continue
|
|
columns = line.split("\t")
|
|
byte = int(columns[0], 16)
|
|
if byte > 0xFF:
|
|
raise TableError(f"{path.name}: byte {byte:#x} out of range")
|
|
if byte in seen:
|
|
raise TableError(f"{path.name}: byte {byte:#04x} listed twice")
|
|
seen.add(byte)
|
|
if len(columns) >= 2 and columns[1].strip().startswith("0x"):
|
|
table[byte] = int(columns[1], 16)
|
|
if len(seen) != 256:
|
|
missing = sorted(set(range(256)) - seen)
|
|
raise TableError(f"{path.name}: bytes never listed: {missing}")
|
|
defined = [scalar for scalar in table if scalar is not None]
|
|
if len(defined) != len(set(defined)):
|
|
raise TableError(f"{path.name}: table is not a bijection")
|
|
return table
|
|
|
|
|
|
def rust_entry(scalar: int | None) -> str:
|
|
"""Render one table slot as a Rust ``Option<char>`` literal."""
|
|
if scalar is None:
|
|
return "None"
|
|
ch = chr(scalar)
|
|
if ch.isprintable() and ch not in ("'", "\\"):
|
|
return f"Some('{ch}')"
|
|
return f"Some('\\u{{{scalar:04X}}}')"
|
|
|
|
|
|
def render_table(code_page: int, table: Sequence[int | None]) -> str:
|
|
"""Render one code page as a Rust constant, eight entries per line."""
|
|
lines = [
|
|
f"/// Code page {code_page}, byte value to Unicode scalar.",
|
|
f"pub(crate) const CP{code_page}: [Option<char>; 256] = [",
|
|
]
|
|
for row_start in range(0, 256, 8):
|
|
row = ", ".join(rust_entry(table[i]) for i in range(row_start, row_start + 8))
|
|
lines.append(f" {row}, // {row_start:#04x}-{row_start + 7:#04x}")
|
|
lines.append("];\n")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def main() -> None:
|
|
"""Regenerate src/tables.rs from every table in CODE_PAGES."""
|
|
rendered = [HEADER]
|
|
for code_page in CODE_PAGES:
|
|
table = parse_table(DATA_DIR / f"CP{code_page}.TXT")
|
|
rendered.append(render_table(code_page, table))
|
|
OUTPUT.write_text("\n".join(rendered), encoding="utf-8")
|
|
subprocess.run(["rustfmt", "--edition", "2021", str(OUTPUT)], check=True)
|
|
print(f"wrote {OUTPUT} ({len(CODE_PAGES)} tables)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|