movq/msdos-encodings
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add the Windows ANSI family: 874 Thai and 1250-1258 as WindowsEncoding

Browse Source 
The ten single-byte ANSI pages join as a second enum alongside
DosEncoding, mirroring its API. Errors now carry a public Encoding
wrapper (Dos | Windows) so both families share the single-byte codec
functions; Encoding::from_code_page resolves a raw code page number
across families, DOS first.

data/ is reorganized into pc/, windows/, and bestfit/, and the
generated single-byte tables split into dos.rs and ansi.rs. Microsoft
publishes an identical Thai table for both families, so DosEncoding::
Cp874 and WindowsEncoding::Cp874 agree byte-for-byte (pinned by test).
This commit is contained in:
Claude Fable 5
2026-07-19 12:40:36 -05:00
parent 518250da2c
commit 2fe53fdee4
40 changed files with 5857 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
tools/gen_tables.py
View File

@@ -5,14 +5,15 @@
"""Generate src/tables/ from the Microsoft mapping tables in data/.
The inputs are the canonical Microsoft code page tables published by the
Unicode Consortium (https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/,
``PC/`` for the single-byte DOS pages and ``WINDOWS/`` for the double-byte
CJK pages).
Unicode Consortium (https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/):
``PC/`` (vendored in ``data/pc/``) for the single-byte DOS pages and
``WINDOWS/`` (vendored in ``data/windows/``) for the Windows ANSI pages and
the double-byte CJK pages.
Single-byte pages become full 256-entry ``[Option<char>; 256]`` tables:
MS-DOS pages are not uniformly ASCII in the low half (CP864 maps 0x25 to
ARABIC PERCENT SIGN) and several pages leave byte values undefined
(CP857/864/869/874).
(CP857/864/869/874, most of the ANSI pages).
Double-byte pages become a 256-entry single-byte/lead-byte table plus a pair
of sorted ``(code, scalar)`` arrays for binary search. Where several codes
@@ -33,13 +34,19 @@ from collections.abc import Sequence
from dataclasses import dataclass
from pathlib import Path
SINGLE_BYTE_PAGES: Sequence[int] = (
DOS_PAGES: Sequence[int] = (
437, 737, 775, 850, 852, 855, 857, 860,
861, 862, 863, 864, 865, 866, 869, 874,
)
ANSI_PAGES: Sequence[int] = (
874, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258,
)
DOUBLE_BYTE_PAGES: Sequence[int] = (932, 936, 949, 950)
DATA_DIR = Path(__file__).resolve().parent.parent / "data"
PC_DIR = DATA_DIR / "pc"
WINDOWS_DIR = DATA_DIR / "windows"
BESTFIT_DIR = DATA_DIR / "bestfit"
OUTPUT_DIR = Path(__file__).resolve().parent.parent / "src" / "tables"
GENERATED_NOTE = """\
@@ -316,27 +323,39 @@ def render_double_byte_page(
)
def main() -> None:
"""Regenerate src/tables/ from every configured code page."""
outputs: list[Path] = []
def generate_single_byte_module(
output_name: str, source_dir: Path, code_pages: Sequence[int]
) -> Path:
"""Render a family of single-byte pages into one tables module.
single_rendered = [GENERATED_NOTE]
for code_page in SINGLE_BYTE_PAGES:
table = parse_table(DATA_DIR / f"CP{code_page}.TXT")
Raises:
TableError: If any page skips a byte value or contains DBCS content.
"""
rendered = [GENERATED_NOTE]
for code_page in code_pages:
table = parse_table(source_dir / f"CP{code_page}.TXT")
if table.listed_bytes != set(range(256)):
missing = sorted(set(range(256)) - table.listed_bytes)
raise TableError(f"CP{code_page}: bytes never listed: {missing}")
if table.leads or table.pairs:
raise TableError(f"CP{code_page}: unexpected DBCS content")
check_bijection(f"CP{code_page}", table)
single_rendered.append(render_single_byte_page(code_page, table))
single_path = OUTPUT_DIR / "single.rs"
single_path.write_text("\n".join(single_rendered), encoding="utf-8")
outputs.append(single_path)
rendered.append(render_single_byte_page(code_page, table))
path = OUTPUT_DIR / output_name
path.write_text("\n".join(rendered), encoding="utf-8")
return path
def main() -> None:
"""Regenerate src/tables/ from every configured code page."""
outputs = [
generate_single_byte_module("dos.rs", PC_DIR, DOS_PAGES),
generate_single_byte_module("ansi.rs", WINDOWS_DIR, ANSI_PAGES),
]
for code_page in DOUBLE_BYTE_PAGES:
table = parse_table(DATA_DIR / f"CP{code_page}.TXT")
wctable = parse_wctable(DATA_DIR / f"bestfit{code_page}.txt")
table = parse_table(WINDOWS_DIR / f"CP{code_page}.TXT")
wctable = parse_wctable(BESTFIT_DIR / f"bestfit{code_page}.txt")
encode = resolve_encode(f"CP{code_page}", table, wctable)
page_path = OUTPUT_DIR / f"cp{code_page}.rs"
page_path.write_text(