188 lines
6.6 KiB
Python
188 lines
6.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
build_index.py — 把 Unispect dump.txt 流式解析成 SQLite 结构化索引。
|
|
|
|
用法:
|
|
python build_index.py [dump.txt路径] [输出db路径]
|
|
|
|
特性:
|
|
- 流式读取，绝不把整个文件读入内存（文件 7MB / 15.8 万行也毫无压力）
|
|
- 表 types: id, kind, name, base_type, line_no, src_order
|
|
- 表 fields: id, type_id, offset_hex, offset_dec, is_static, is_const,
|
|
name, field_type, line_no
|
|
- FTS5 全文索引: types_fts (name/base_type), fields_fts (name/field_type)
|
|
- 枚举常量(C)与静态字段(S)独立标记
|
|
"""
|
|
import io
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
import time
|
|
|
|
SRC_DEFAULT = r"F:\desktop\Release\dump.txt"
|
|
DB_DEFAULT = r"F:\desktop\CFramework\dump_index.db"
|
|
|
|
TYPE_HEAD_RE = re.compile(
|
|
r"^\[(Class|Interface|Enum|Struct)\]\s+(.+?)(?:\s*:\s*(.+))?$"
|
|
)
|
|
FIELD_RE = re.compile(
|
|
r"^\s+\[([0-9A-Fa-f]+)\](?:\[([SC])\])?\s+(.+?)\s*:\s*(.+)$"
|
|
)
|
|
# 兜底: 缩进但没匹配字段正则的行（异常行, 记录到 log 便于排查, 不影响索引）
|
|
INDENTED_RE = re.compile(r"^\s+\S")
|
|
|
|
KIND_MAP = {"Class": "class", "Interface": "interface", "Enum": "enum", "Struct": "struct"}
|
|
|
|
|
|
def clean_name(name: str) -> str:
|
|
"""去掉 Unispect 的 `-.` 内部类前缀与泛型参数尾巴, 得到可读名。"""
|
|
n = name.strip()
|
|
if n.startswith("-."):
|
|
n = n[2:]
|
|
# 去掉泛型 `...<...>` 的尖括号部分用于短名
|
|
short = re.split(r"[<`]", n, maxsplit=1)[0]
|
|
return n, short
|
|
|
|
|
|
def build(src: str, db: str) -> None:
|
|
if os.path.exists(db):
|
|
os.remove(db)
|
|
|
|
t0 = time.time()
|
|
con = sqlite3.connect(db)
|
|
con.execute("PRAGMA journal_mode=OFF")
|
|
con.execute("PRAGMA synchronous=OFF")
|
|
con.execute("PRAGMA temp_store=MEMORY")
|
|
cur = con.cursor()
|
|
|
|
cur.executescript(
|
|
"""
|
|
CREATE TABLE types (
|
|
id INTEGER PRIMARY KEY,
|
|
kind TEXT NOT NULL,
|
|
name TEXT NOT NULL, -- 原始名 (含 -. / 泛型)
|
|
short TEXT NOT NULL, -- 可读短名
|
|
base_type TEXT,
|
|
line_no INTEGER,
|
|
src_order INTEGER
|
|
);
|
|
CREATE TABLE fields (
|
|
id INTEGER PRIMARY KEY,
|
|
type_id INTEGER NOT NULL REFERENCES types(id),
|
|
offset_hex TEXT NOT NULL,
|
|
offset_dec INTEGER NOT NULL,
|
|
is_static INTEGER NOT NULL DEFAULT 0,
|
|
is_const INTEGER NOT NULL DEFAULT 0,
|
|
name TEXT NOT NULL,
|
|
field_type TEXT,
|
|
line_no INTEGER
|
|
);
|
|
CREATE INDEX idx_fields_type ON fields(type_id);
|
|
CREATE INDEX idx_fields_name ON fields(name);
|
|
CREATE INDEX idx_types_name ON types(name);
|
|
CREATE INDEX idx_types_short ON types(short);
|
|
CREATE INDEX idx_types_base ON types(base_type);
|
|
|
|
CREATE VIRTUAL TABLE types_fts USING fts5(
|
|
name, short, base_type, content='types', content_rowid='id'
|
|
);
|
|
CREATE VIRTUAL TABLE fields_fts USING fts5(
|
|
name, field_type, content='fields', content_rowid='id'
|
|
);
|
|
"""
|
|
)
|
|
|
|
type_id = 0
|
|
field_id = 0
|
|
current = None # (type_id, kind, name)
|
|
type_count = 0
|
|
field_count = 0
|
|
anomaly_log = []
|
|
|
|
with io.open(src, "r", encoding="utf-8", errors="replace", newline="\n") as f:
|
|
for lineno, raw in enumerate(f, 1):
|
|
line = raw.rstrip("\r\n")
|
|
t = line.strip()
|
|
if not t:
|
|
continue
|
|
m = TYPE_HEAD_RE.match(t)
|
|
if m:
|
|
kind_raw, name, base = m.group(1), m.group(2), m.group(3)
|
|
type_id += 1
|
|
name_full, short = clean_name(name)
|
|
base_s = (base or "").strip()
|
|
cur.execute(
|
|
"INSERT INTO types (id, kind, name, short, base_type, line_no, src_order) "
|
|
"VALUES (?,?,?,?,?,?,?)",
|
|
(type_id, KIND_MAP[kind_raw], name_full, short, base_s,
|
|
lineno, type_id),
|
|
)
|
|
cur.execute(
|
|
"INSERT INTO types_fts (rowid, name, short, base_type) VALUES (?,?,?,?)",
|
|
(type_id, name_full, short, base_s),
|
|
)
|
|
current = type_id
|
|
type_count += 1
|
|
continue
|
|
|
|
fm = FIELD_RE.match(line)
|
|
if fm and current is not None:
|
|
off_hex = fm.group(1)
|
|
flag = fm.group(2) or ""
|
|
fname = fm.group(3).strip()
|
|
ftype = fm.group(4).strip()
|
|
field_id += 1
|
|
cur.execute(
|
|
"INSERT INTO fields (id, type_id, offset_hex, offset_dec, is_static, "
|
|
"is_const, name, field_type, line_no) VALUES (?,?,?,?,?,?,?,?,?)",
|
|
(field_id, current, off_hex, int(off_hex, 16),
|
|
1 if flag == "S" else 0,
|
|
1 if flag == "C" else 0,
|
|
fname, ftype, lineno),
|
|
)
|
|
cur.execute(
|
|
"INSERT INTO fields_fts (rowid, name, field_type) VALUES (?,?,?)",
|
|
(field_id, fname, ftype),
|
|
)
|
|
field_count += 1
|
|
continue
|
|
|
|
# 异常行: 缩进但解析不了, 或类型头后跟的杂行
|
|
if INDENTED_RE.match(line) and current is not None:
|
|
if len(anomaly_log) < 50:
|
|
anomaly_log.append(f"L{lineno}: {t[:80]!r}")
|
|
|
|
# 重建 FTS 索引内容表一致性
|
|
cur.execute("INSERT INTO types_fts(types_fts) VALUES('rebuild')")
|
|
cur.execute("INSERT INTO fields_fts(fields_fts) VALUES('rebuild')")
|
|
con.commit()
|
|
|
|
elapsed = time.time() - t0
|
|
print(f"types = {type_count}")
|
|
print(f"fields = {field_count}")
|
|
print(f"anomaly = {len(anomaly_log)}")
|
|
for a in anomaly_log:
|
|
print(" !", a)
|
|
print(f"elapsed = {elapsed:.2f}s")
|
|
print(f"db = {db} ({os.path.getsize(db) / 1024:.0f} KB)")
|
|
|
|
# 内置快速校验
|
|
print("\n-- sanity checks --")
|
|
for q, label in [
|
|
("SELECT kind, COUNT(*) FROM types GROUP BY kind", "types by kind"),
|
|
("SELECT COUNT(*) FROM types WHERE base_type IS NULL OR base_type=''", "types w/o base"),
|
|
("SELECT COUNT(*) FROM fields WHERE is_static=1", "static fields"),
|
|
("SELECT COUNT(*) FROM fields WHERE is_const=1", "const fields"),
|
|
]:
|
|
rows = cur.execute(q).fetchall()
|
|
print(f"{label}: {rows}")
|
|
|
|
con.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
src = sys.argv[1] if len(sys.argv) > 1 else SRC_DEFAULT
|
|
db = sys.argv[2] if len(sys.argv) > 2 else DB_DEFAULT
|
|
build(src, db)
|