518250da2c
Code pages 932 (Shift-JIS), 936 (GBK), 949 (Unified Hangul), and 950 (Big5), generated from the Unicode Consortium's Microsoft WINDOWS tables. Each page carries a three-state single-byte table (map / lead / undefined) plus sorted pair arrays binary-searched in both directions. Where several codes decode to one scalar (932's NEC/IBM overlap, 950's duplicated box-drawing rows), the encoder's winner is resolved from Microsoft's own bestfit WCTABLE — used only for duplicate resolution, never to import lossy best-fit mappings. The full decode direction was cross-validated against Python's cp932/936/949/950 codecs: zero mismatches over 60k+ pairs. Decode gains TruncatedPair and UndefinedPair failure modes; decode_lossy emits one U+FFFD per failed pair. Both feature configurations are clippy-clean and tested.
356 lines
13 KiB
Python
356 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
# /// script
|
|
# requires-python = ">=3.10"
|
|
# ///
|
|
"""Generate src/tables/ from the Microsoft mapping tables in data/.
|
|
|
|
The inputs are the canonical Microsoft code page tables published by the
|
|
Unicode Consortium (https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/,
|
|
``PC/`` for the single-byte DOS pages and ``WINDOWS/`` for the double-byte
|
|
CJK pages).
|
|
|
|
Single-byte pages become full 256-entry ``[Option<char>; 256]`` tables:
|
|
MS-DOS pages are not uniformly ASCII in the low half (CP864 maps 0x25 to
|
|
ARABIC PERCENT SIGN) and several pages leave byte values undefined
|
|
(CP857/864/869/874).
|
|
|
|
Double-byte pages become a 256-entry single-byte/lead-byte table plus a pair
|
|
of sorted ``(code, scalar)`` arrays for binary search. Where several codes
|
|
decode to one scalar (CP932's NEC/IBM overlap, CP950's box-drawing rows),
|
|
the encoder's choice is resolved from Microsoft's own ``bestfit*.txt``
|
|
WCTABLE — used *only* to pick among duplicate candidates, never to import
|
|
lossy best-fit mappings.
|
|
|
|
Run from the crate root: ``uv run tools/gen_tables.py``
|
|
(needs ``rustfmt`` on PATH).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import subprocess
|
|
from collections.abc import Sequence
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
SINGLE_BYTE_PAGES: Sequence[int] = (
|
|
437, 737, 775, 850, 852, 855, 857, 860,
|
|
861, 862, 863, 864, 865, 866, 869, 874,
|
|
)
|
|
DOUBLE_BYTE_PAGES: Sequence[int] = (932, 936, 949, 950)
|
|
|
|
DATA_DIR = Path(__file__).resolve().parent.parent / "data"
|
|
OUTPUT_DIR = Path(__file__).resolve().parent.parent / "src" / "tables"
|
|
|
|
GENERATED_NOTE = """\
|
|
//! Generated by `tools/gen_tables.py` — do not edit by hand.
|
|
//!
|
|
//! Source: the Unicode Consortium's Microsoft mapping tables, vendored in
|
|
//! `data/`.
|
|
"""
|
|
|
|
SURROGATE_RANGE = range(0xD800, 0xE000)
|
|
|
|
|
|
class TableError(Exception):
|
|
"""A mapping table failed to parse or violated an invariant."""
|
|
|
|
|
|
@dataclass
|
|
class CodePageTable:
|
|
"""One parsed Microsoft mapping table.
|
|
|
|
Attributes:
|
|
single: Byte value to Unicode scalar for single-byte codes.
|
|
leads: Byte values marked ``DBCS LEAD BYTE`` (empty for single-byte
|
|
pages).
|
|
pairs: 16-bit lead<<8|trail code to Unicode scalar.
|
|
listed_bytes: Every byte value the file listed, mapped or not.
|
|
"""
|
|
|
|
single: dict[int, int]
|
|
leads: set[int]
|
|
pairs: dict[int, int]
|
|
listed_bytes: set[int]
|
|
|
|
|
|
def read_mapping_lines(path: Path) -> list[str]:
|
|
"""Read a mapping file and return its data lines (those starting 0x).
|
|
|
|
Raises:
|
|
TableError: If the file is missing or non-ASCII (the lone stray byte
|
|
in canonical data is a DOS EOF marker, 0x1A, which is still ASCII).
|
|
"""
|
|
try:
|
|
text = path.read_text(encoding="ascii")
|
|
except (OSError, UnicodeDecodeError) as error:
|
|
raise TableError(f"{path.name}: unreadable mapping table") from error
|
|
return [line.strip() for line in text.splitlines() if line.strip().startswith("0x")]
|
|
|
|
|
|
def parse_table(path: Path) -> CodePageTable:
|
|
"""Parse one CP*.TXT into single-byte, lead-byte, and pair mappings.
|
|
|
|
Raises:
|
|
TableError: If a code is listed twice, a single byte is out of range,
|
|
a scalar is a surrogate or beyond the BMP, or byte 0x00 appears
|
|
as a DBCS trail (which would break NUL-terminated decoding).
|
|
"""
|
|
table = CodePageTable(single={}, leads=set(), pairs={}, listed_bytes=set())
|
|
seen: set[int] = set()
|
|
for line in read_mapping_lines(path):
|
|
columns = line.split("\t")
|
|
code = int(columns[0], 16)
|
|
if code in seen:
|
|
raise TableError(f"{path.name}: code {code:#06x} listed twice")
|
|
seen.add(code)
|
|
if code > 0xFFFF:
|
|
raise TableError(f"{path.name}: code {code:#x} out of range")
|
|
if code <= 0xFF:
|
|
table.listed_bytes.add(code)
|
|
mapped = len(columns) >= 2 and columns[1].strip().startswith("0x")
|
|
if not mapped:
|
|
if code <= 0xFF and "DBCS LEAD BYTE" in line:
|
|
table.leads.add(code)
|
|
continue
|
|
scalar = int(columns[1], 16)
|
|
if scalar in SURROGATE_RANGE or scalar > 0xFFFF:
|
|
raise TableError(f"{path.name}: scalar {scalar:#06x} not a BMP char")
|
|
if code <= 0xFF:
|
|
table.single[code] = scalar
|
|
else:
|
|
if code & 0xFF == 0:
|
|
raise TableError(f"{path.name}: NUL trail byte in {code:#06x}")
|
|
table.pairs[code] = scalar
|
|
unmarked_leads = {code >> 8 for code in table.pairs} - table.leads
|
|
if unmarked_leads:
|
|
raise TableError(
|
|
f"{path.name}: pairs under bytes not marked DBCS LEAD BYTE: "
|
|
f"{[hex(lead) for lead in sorted(unmarked_leads)]}"
|
|
)
|
|
return table
|
|
|
|
|
|
def parse_wctable(path: Path) -> dict[int, int]:
|
|
"""Parse the WCTABLE (Unicode to codepage) section of a bestfit file.
|
|
|
|
Raises:
|
|
TableError: If the file is missing or has no WCTABLE section.
|
|
"""
|
|
try:
|
|
# Comments embed raw codepage-encoded glyph bytes; latin-1 accepts
|
|
# any byte and the data lines we match are pure ASCII.
|
|
text = path.read_text(encoding="latin-1")
|
|
except OSError as error:
|
|
raise TableError(f"{path.name}: unreadable bestfit table") from error
|
|
_, marker, rest = text.partition("WCTABLE")
|
|
if not marker:
|
|
raise TableError(f"{path.name}: no WCTABLE section")
|
|
section = rest.split("ENDCODEPAGE")[0]
|
|
entry = re.compile(r"^0x([0-9a-fA-F]{4})\s+0x([0-9a-fA-F]+)")
|
|
table: dict[int, int] = {}
|
|
for line in section.splitlines():
|
|
if match := entry.match(line.strip()):
|
|
table[int(match.group(1), 16)] = int(match.group(2), 16)
|
|
return table
|
|
|
|
|
|
def check_bijection(path_name: str, table: CodePageTable) -> None:
|
|
"""Require every scalar to have exactly one code (single-byte pages).
|
|
|
|
Raises:
|
|
TableError: If two codes map to the same scalar.
|
|
"""
|
|
scalars = list(table.single.values()) + list(table.pairs.values())
|
|
if len(scalars) != len(set(scalars)):
|
|
raise TableError(f"{path_name}: table is not a bijection")
|
|
|
|
|
|
def resolve_encode(
|
|
path_name: str, table: CodePageTable, wctable: dict[int, int]
|
|
) -> dict[int, int]:
|
|
"""Choose one code per scalar, resolving duplicates via the WCTABLE.
|
|
|
|
Args:
|
|
path_name: The mapping file name, for error messages.
|
|
table: The parsed decode-direction table.
|
|
wctable: Microsoft's Unicode-to-codepage table for this page.
|
|
|
|
Returns:
|
|
Scalar to pair-code for the double-byte portion only (single-byte
|
|
codes never collide with pairs in these tables, which is asserted).
|
|
|
|
Raises:
|
|
TableError: If a duplicate set is not settled by the WCTABLE, mixes
|
|
single-byte and pair codes (the Rust encoder tries the single
|
|
table first, so a pair winner would be unreachable), or has a
|
|
single-byte winner that is not the lowest byte (the Rust encoder
|
|
scans upward, so any other winner would be shadowed).
|
|
"""
|
|
candidates: dict[int, list[int]] = {}
|
|
for code, scalar in list(table.single.items()) + list(table.pairs.items()):
|
|
candidates.setdefault(scalar, []).append(code)
|
|
encode: dict[int, int] = {}
|
|
for scalar, codes in candidates.items():
|
|
if len(codes) == 1:
|
|
chosen = codes[0]
|
|
else:
|
|
if any(c <= 0xFF for c in codes) and any(c > 0xFF for c in codes):
|
|
raise TableError(
|
|
f"{path_name}: U+{scalar:04X} mixes single and pair codes "
|
|
f"({[hex(c) for c in codes]})"
|
|
)
|
|
chosen = wctable.get(scalar, -1)
|
|
if chosen not in codes:
|
|
raise TableError(
|
|
f"{path_name}: WCTABLE does not settle duplicate scalar "
|
|
f"U+{scalar:04X} (candidates {[hex(c) for c in codes]})"
|
|
)
|
|
if chosen <= 0xFF and chosen != min(codes):
|
|
raise TableError(
|
|
f"{path_name}: U+{scalar:04X} single-byte winner "
|
|
f"{chosen:#04x} is not the lowest candidate"
|
|
)
|
|
if chosen <= 0xFF:
|
|
continue # single-byte encoding is derived from the single table
|
|
encode[scalar] = chosen
|
|
return encode
|
|
|
|
|
|
def rust_char(scalar: int) -> str:
|
|
"""Render a Unicode scalar as a Rust char literal."""
|
|
ch = chr(scalar)
|
|
if ch.isprintable() and ch not in ("'", "\\"):
|
|
return f"'{ch}'"
|
|
return f"'\\u{{{scalar:04X}}}'"
|
|
|
|
|
|
def render_single_byte_page(code_page: int, table: CodePageTable) -> str:
|
|
"""Render one single-byte code page as an ``[Option<char>; 256]``."""
|
|
lines = [
|
|
f"/// Code page {code_page}, byte value to Unicode scalar.",
|
|
f"pub(crate) const CP{code_page}: [Option<char>; 256] = [",
|
|
]
|
|
for row_start in range(0, 256, 8):
|
|
entries = []
|
|
for byte in range(row_start, row_start + 8):
|
|
scalar = table.single.get(byte)
|
|
entries.append("None" if scalar is None else f"Some({rust_char(scalar)})")
|
|
lines.append(f" {', '.join(entries)}, // {row_start:#04x}-{row_start + 7:#04x}")
|
|
lines.append("];\n")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def render_pair_array(name: str, doc: str, pairs: Sequence[tuple[int, int]]) -> str:
|
|
"""Render a sorted ``[(u16, char); N]`` static, eight entries per line."""
|
|
lines = [f"/// {doc}", f"static {name}: [(u16, char); {len(pairs)}] = ["]
|
|
for row_start in range(0, len(pairs), 8):
|
|
row = pairs[row_start : row_start + 8]
|
|
rendered = ", ".join(f"({code:#06x}, {rust_char(scalar)})" for code, scalar in row)
|
|
lines.append(f" {rendered},")
|
|
lines.append("];\n")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def render_encode_array(name: str, doc: str, pairs: Sequence[tuple[int, int]]) -> str:
|
|
"""Render a sorted ``[(char, u16); N]`` static, eight entries per line."""
|
|
lines = [f"/// {doc}", f"static {name}: [(char, u16); {len(pairs)}] = ["]
|
|
for row_start in range(0, len(pairs), 8):
|
|
row = pairs[row_start : row_start + 8]
|
|
rendered = ", ".join(f"({rust_char(scalar)}, {code:#06x})" for scalar, code in row)
|
|
lines.append(f" {rendered},")
|
|
lines.append("];\n")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def render_double_byte_page(
|
|
code_page: int, table: CodePageTable, encode: dict[int, int]
|
|
) -> str:
|
|
"""Render one DBCS page as a module body with single/decode/encode tables."""
|
|
single_lines = [
|
|
"/// Single-byte slots: mapped char, DBCS lead byte, or undefined.",
|
|
"const SINGLE: [SingleEntry; 256] = [",
|
|
]
|
|
for row_start in range(0, 256, 8):
|
|
entries = []
|
|
for byte in range(row_start, row_start + 8):
|
|
if byte in table.leads:
|
|
entries.append("L")
|
|
elif (scalar := table.single.get(byte)) is not None:
|
|
entries.append(f"M({rust_char(scalar)})")
|
|
else:
|
|
entries.append("U")
|
|
single_lines.append(
|
|
f" {', '.join(entries)}, // {row_start:#04x}-{row_start + 7:#04x}"
|
|
)
|
|
single_lines.append("];\n")
|
|
|
|
decode_pairs = sorted(table.pairs.items())
|
|
encode_pairs = sorted(encode.items())
|
|
return "\n".join(
|
|
[
|
|
GENERATED_NOTE,
|
|
"use super::SingleEntry::{self, Lead as L, Map as M, Undefined as U};",
|
|
"use super::DbcsPage;",
|
|
"",
|
|
f"/// Code page {code_page}.",
|
|
f"pub(crate) static CP{code_page}: DbcsPage = DbcsPage {{",
|
|
" single: &SINGLE,",
|
|
" decode: &DECODE,",
|
|
" encode: &ENCODE,",
|
|
"};",
|
|
"",
|
|
"\n".join(single_lines),
|
|
render_pair_array(
|
|
"DECODE",
|
|
"Every defined pair, sorted by 16-bit code (lead<<8 | trail).",
|
|
decode_pairs,
|
|
),
|
|
render_encode_array(
|
|
"ENCODE",
|
|
"One pair per scalar, sorted by scalar; duplicates resolved per "
|
|
"Microsoft's bestfit WCTABLE.",
|
|
encode_pairs,
|
|
),
|
|
]
|
|
)
|
|
|
|
|
|
def main() -> None:
|
|
"""Regenerate src/tables/ from every configured code page."""
|
|
outputs: list[Path] = []
|
|
|
|
single_rendered = [GENERATED_NOTE]
|
|
for code_page in SINGLE_BYTE_PAGES:
|
|
table = parse_table(DATA_DIR / f"CP{code_page}.TXT")
|
|
if table.listed_bytes != set(range(256)):
|
|
missing = sorted(set(range(256)) - table.listed_bytes)
|
|
raise TableError(f"CP{code_page}: bytes never listed: {missing}")
|
|
if table.leads or table.pairs:
|
|
raise TableError(f"CP{code_page}: unexpected DBCS content")
|
|
check_bijection(f"CP{code_page}", table)
|
|
single_rendered.append(render_single_byte_page(code_page, table))
|
|
single_path = OUTPUT_DIR / "single.rs"
|
|
single_path.write_text("\n".join(single_rendered), encoding="utf-8")
|
|
outputs.append(single_path)
|
|
|
|
for code_page in DOUBLE_BYTE_PAGES:
|
|
table = parse_table(DATA_DIR / f"CP{code_page}.TXT")
|
|
wctable = parse_wctable(DATA_DIR / f"bestfit{code_page}.txt")
|
|
encode = resolve_encode(f"CP{code_page}", table, wctable)
|
|
page_path = OUTPUT_DIR / f"cp{code_page}.rs"
|
|
page_path.write_text(
|
|
render_double_byte_page(code_page, table, encode), encoding="utf-8"
|
|
)
|
|
outputs.append(page_path)
|
|
|
|
subprocess.run(
|
|
["rustfmt", "--edition", "2021", *map(str, outputs)], check=True
|
|
)
|
|
for path in outputs:
|
|
print(f"wrote {path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|