# -*- coding: utf-8 -*- """ build_index.py — 把 Unispect dump.txt 流式解析成 SQLite 结构化索引。 用法: python build_index.py [dump.txt路径] [输出db路径] 特性: - 流式读取，绝不把整个文件读入内存（文件 7MB / 15.8 万行也毫无压力） - 表 types: id, kind, name, base_type, line_no, src_order - 表 fields: id, type_id, offset_hex, offset_dec, is_static, is_const, name, field_type, line_no - FTS5 全文索引: types_fts (name/base_type), fields_fts (name/field_type) - 枚举常量(C)与静态字段(S)独立标记 """ import io import os import re import sqlite3 import sys import time SRC_DEFAULT = r"F:\desktop\Release\dump.txt" DB_DEFAULT = r"F:\desktop\CFramework\dump_index.db" TYPE_HEAD_RE = re.compile( r"^\[(Class|Interface|Enum|Struct)\]\s+(.+?)(?:\s*:\s*(.+))?$" ) FIELD_RE = re.compile( r"^\s+\[([0-9A-Fa-f]+)\](?:\[([SC])\])?\s+(.+?)\s*:\s*(.+)$" ) # 兜底: 缩进但没匹配字段正则的行（异常行, 记录到 log 便于排查, 不影响索引） INDENTED_RE = re.compile(r"^\s+\S") KIND_MAP = {"Class": "class", "Interface": "interface", "Enum": "enum", "Struct": "struct"} def clean_name(name: str) -> str: """去掉 Unispect 的 `-.` 内部类前缀与泛型参数尾巴, 得到可读名。""" n = name.strip() if n.startswith("-."): n = n[2:] # 去掉泛型 `...<...>` 的尖括号部分用于短名 short = re.split(r"[<`]", n, maxsplit=1)[0] return n, short def build(src: str, db: str) -> None: if os.path.exists(db): os.remove(db) t0 = time.time() con = sqlite3.connect(db) con.execute("PRAGMA journal_mode=OFF") con.execute("PRAGMA synchronous=OFF") con.execute("PRAGMA temp_store=MEMORY") cur = con.cursor() cur.executescript( """ CREATE TABLE types ( id INTEGER PRIMARY KEY, kind TEXT NOT NULL, name TEXT NOT NULL, -- 原始名 (含 -. / 泛型) short TEXT NOT NULL, -- 可读短名 base_type TEXT, line_no INTEGER, src_order INTEGER ); CREATE TABLE fields ( id INTEGER PRIMARY KEY, type_id INTEGER NOT NULL REFERENCES types(id), offset_hex TEXT NOT NULL, offset_dec INTEGER NOT NULL, is_static INTEGER NOT NULL DEFAULT 0, is_const INTEGER NOT NULL DEFAULT 0, name TEXT NOT NULL, field_type TEXT, line_no INTEGER ); CREATE INDEX idx_fields_type ON fields(type_id); CREATE INDEX idx_fields_name ON fields(name); CREATE INDEX idx_types_name ON types(name); CREATE INDEX idx_types_short ON types(short); CREATE INDEX idx_types_base ON types(base_type); CREATE VIRTUAL TABLE types_fts USING fts5( name, short, base_type, content='types', content_rowid='id' ); CREATE VIRTUAL TABLE fields_fts USING fts5( name, field_type, content='fields', content_rowid='id' ); """ ) type_id = 0 field_id = 0 current = None # (type_id, kind, name) type_count = 0 field_count = 0 anomaly_log = [] with io.open(src, "r", encoding="utf-8", errors="replace", newline="\n") as f: for lineno, raw in enumerate(f, 1): line = raw.rstrip("\r\n") t = line.strip() if not t: continue m = TYPE_HEAD_RE.match(t) if m: kind_raw, name, base = m.group(1), m.group(2), m.group(3) type_id += 1 name_full, short = clean_name(name) base_s = (base or "").strip() cur.execute( "INSERT INTO types (id, kind, name, short, base_type, line_no, src_order) " "VALUES (?,?,?,?,?,?,?)", (type_id, KIND_MAP[kind_raw], name_full, short, base_s, lineno, type_id), ) cur.execute( "INSERT INTO types_fts (rowid, name, short, base_type) VALUES (?,?,?,?)", (type_id, name_full, short, base_s), ) current = type_id type_count += 1 continue fm = FIELD_RE.match(line) if fm and current is not None: off_hex = fm.group(1) flag = fm.group(2) or "" fname = fm.group(3).strip() ftype = fm.group(4).strip() field_id += 1 cur.execute( "INSERT INTO fields (id, type_id, offset_hex, offset_dec, is_static, " "is_const, name, field_type, line_no) VALUES (?,?,?,?,?,?,?,?,?)", (field_id, current, off_hex, int(off_hex, 16), 1 if flag == "S" else 0, 1 if flag == "C" else 0, fname, ftype, lineno), ) cur.execute( "INSERT INTO fields_fts (rowid, name, field_type) VALUES (?,?,?)", (field_id, fname, ftype), ) field_count += 1 continue # 异常行: 缩进但解析不了, 或类型头后跟的杂行 if INDENTED_RE.match(line) and current is not None: if len(anomaly_log) < 50: anomaly_log.append(f"L{lineno}: {t[:80]!r}") # 重建 FTS 索引内容表一致性 cur.execute("INSERT INTO types_fts(types_fts) VALUES('rebuild')") cur.execute("INSERT INTO fields_fts(fields_fts) VALUES('rebuild')") con.commit() elapsed = time.time() - t0 print(f"types = {type_count}") print(f"fields = {field_count}") print(f"anomaly = {len(anomaly_log)}") for a in anomaly_log: print(" !", a) print(f"elapsed = {elapsed:.2f}s") print(f"db = {db} ({os.path.getsize(db) / 1024:.0f} KB)") # 内置快速校验 print("\n-- sanity checks --") for q, label in [ ("SELECT kind, COUNT(*) FROM types GROUP BY kind", "types by kind"), ("SELECT COUNT(*) FROM types WHERE base_type IS NULL OR base_type=''", "types w/o base"), ("SELECT COUNT(*) FROM fields WHERE is_static=1", "static fields"), ("SELECT COUNT(*) FROM fields WHERE is_const=1", "const fields"), ]: rows = cur.execute(q).fetchall() print(f"{label}: {rows}") con.close() if __name__ == "__main__": src = sys.argv[1] if len(sys.argv) > 1 else SRC_DEFAULT db = sys.argv[2] if len(sys.argv) > 2 else DB_DEFAULT build(src, db)