diff --git a/DUMP_INDEX.md b/DUMP_INDEX.md new file mode 100644 index 0000000..eb4e550 --- /dev/null +++ b/DUMP_INDEX.md @@ -0,0 +1,227 @@ +# dump.txt 结构化索引 — 使用文档 + +`dump.txt` 是 Unispect v0.2.4.0 生成的 IL2CPP 类布局 dump(15.8 万行 / 7MB)。 +本工具将其解析为 SQLite 数据库 `dump_index.db`,提供毫秒级类型 / 字段 / 偏移检索, +替代对原始文本的暴力搜索。 + +--- + +## 1. 快速开始 + +```powershell +# 构建索引(默认输入 F:\desktop\Release\dump.txt,输出 dump_index.db) +python build_index.py + +# 自定义路径 +python build_index.py D:\other\dump.txt D:\other\dump_index.db + +# 查询统计 +python query_index.py stats +``` + +构建耗时约 1.5 秒,输出: + +``` +types = 15137 # class 11118 / interface 1118 / struct 1998 / enum 903 +fields = 143599 # static 12350 / const 17693 +anomaly = 0 # 无法解析的行数(应为 0) +``` + +--- + +## 2. 数据库结构(schema) + +```sql +CREATE TABLE types ( + id INTEGER PRIMARY KEY, -- 类型 ID,查询时可直接用 + kind TEXT, -- class / interface / enum / struct + name TEXT, -- 原始名(含 -. 前缀与泛型参数,如 -.Player.FirearmController) + short TEXT, -- 可读短名(去掉 -. 前缀与泛型尾巴,如 Player.FirearmController) + base_type TEXT, -- 基类(接口无基类时为空串) + line_no INTEGER, -- 在 dump.txt 中的行号 + src_order INTEGER -- 原始顺序 +); + +CREATE TABLE fields ( + id INTEGER PRIMARY KEY, + type_id INTEGER REFERENCES types(id), -- 所属类型 + offset_hex TEXT, -- 十六进制偏移,如 "1D8" + offset_dec INTEGER, -- 十进制偏移(排序用) + is_static INTEGER, -- 1 = 静态字段 [S] + is_const INTEGER, -- 1 = 常量 [C] + name TEXT, -- 字段名 + field_type TEXT, -- 字段类型 + line_no INTEGER +); + +-- FTS5 全文索引(按 rowid 关联) +CREATE VIRTUAL TABLE types_fts USING fts5(name, short, base_type, ...); +CREATE VIRTUAL TABLE fields_fts USING fts5(name, field_type, ...); +``` + +字段行格式还原(与 dump.txt 一致): + +``` + [偏移][S|C] 字段名 : 字段类型 + [1D8] CoverStruct : -.GStruct13 ← 普通字段 + [00][S] staticVar : Int32 ← 静态字段 + [00][C] TooLittle : System.Int32 ← 枚举常量 / const +``` + +--- + +## 3. 查询命令手册 + +所有命令:`python query_index.py <命令> [关键词] [--limit N]` + +### 3.1 统计总览 + +```powershell +python query_index.py stats +``` + +输出类型总数、各类别数量、字段最多的 TOP10 类型(可直接拿 ID 去展开)。 + +### 3.2 按类型名搜索 + +```powershell +python query_index.py type Player +python query_index.py type "GClass" --limit 50 +``` + +模糊匹配 `name` 与 `short`,按字段数降序排列,便于快速定位"大头"类。 + +### 3.3 按基类搜索(继承关系) + +```powershell +python query_index.py base MonoBehaviour +python query_index.py base "GClass429" +``` + +找出所有直接继承自某基类的类型。 + +### 3.4 全局搜字段名 + +```powershell +python query_index.py field Health +python query_index.py field "k__BackingField" --limit 100 +``` + +跨所有类型搜索字段名,输出含所属类型、偏移、类型。 + +### 3.5 按字段类型搜索 + +```powershell +python query_index.py ftype "List<" +python query_index.py ftype "System.Action" +``` + +找出所有声明为该类型的字段(如所有 `List` 字段)。 + +### 3.6 展开一个类型的全部字段 + +```powershell +python query_index.py show 13764 # 按 ID +python query_index.py show EFT.Player # 按名字(唯一匹配时) +``` + +输出该类型所有字段:偏移、S/C 标记、字段名、类型。按偏移升序。 + +### 3.7 还原原始 dump 格式 + +```powershell +python query_index.py dump 12702 +``` + +按 dump.txt 原样格式输出整个类型块,便于与原文对比或导出。 + +### 3.8 通用选项 + +| 选项 | 说明 | 默认 | +|---|---|---| +| `--limit N` | 搜索结果上限 | 20 | +| `--db PATH` | 指定索引库路径 | `dump_index.db` | + +--- + +## 4. 高级用法:直接查 SQLite + +不经过 CLI,直接用 Python / sqlite3 查询: + +```powershell +python -c " +import sqlite3 +con = sqlite3.connect(r'F:\desktop\CFramework\dump_index.db') +# 1. 找某类型所有字段 +for r in con.execute(''' + SELECT offset_hex, name, field_type FROM fields + WHERE type_id = (SELECT id FROM types WHERE name='EFT.Player') + ORDER BY offset_dec'''): + print(r) +" +``` + +常用 SQL 片段: + +```sql +-- 同偏移字段(union/variant 布局分析) +SELECT f.type_id, t.name, f.offset_hex, f.name, f.field_type +FROM fields f JOIN types t ON t.id=f.type_id +WHERE f.offset_hex='00' AND f.is_static=0 +GROUP BY f.type_id HAVING COUNT(*)>1; + +-- 继承链:所有 EFT.Player 的派生类 +SELECT id, name, base_type FROM types WHERE base_type LIKE '%Player%'; + +-- FTS 快速搜索(token 级,比 LIKE 快) +SELECT t.id, t.name +FROM types_fts JOIN types t ON t.id = types_fts.rowid +WHERE types_fts MATCH 'player'; + +-- 某类型静态字段清单 +SELECT offset_hex, name, field_type FROM fields +WHERE type_id=13764 AND is_static=1 ORDER BY offset_dec; +``` + +--- + +## 5. 标记说明 + +| 标记 | 含义 | +|---|---| +| `[S]` | Static 静态字段(`is_static=1`) | +| `[C]` | Constant 常量 / 枚举值(`is_const=1`) | +| 无标记 | 实例字段 | +| 偏移 | 字段在对象内存布局中的偏移(十六进制),`offset_dec` 为其十进制值 | + +注意:枚举类型的枚举值在 dump 中也是 `[C]` 行(偏移恒为 `00`), +展开 enum 时看到的 `[00][C] XXX : System.Int32` 就是枚举成员。 + +--- + +## 6. 已知限制 + +1. **dump 不含方法**:Unispect 默认只导出字段布局,索引同样只有字段。 + 方法签名 / IL 不在 `dump.txt` 内,无法索引。 +2. **异常字符行**:极少数字段名含控制字符(如 `` ` >j\x01 ``),解析时会被跳过, + 构建日志中 `anomaly` 计数反映数量(当前为 0)。 +3. **同名类型**:不同命名空间下存在同名类型(如多个 `GClass857`), + `show` / `dump` 用名字定位时若多匹配会列出候选,请改用 ID 精确指定。 +4. **`-.` 前缀**:表示 global namespace 或嵌套类,`name` 保留原始形式, + 搜索时 `short` 列已去掉此前缀,两者都参与匹配。 + +--- + +## 7. 常见问题 + +**Q: 为什么我搜 `type GClass857` 出现多个结果?** +A: 同名类型在多处定义(正常现象),用 `show ` 精确定位。 + +**Q: 想重新生成索引?** +A: 直接重跑 `python build_index.py`,会自动覆盖旧库。 + +**Q: 查询报"索引不存在"?** +A: 先运行 `python build_index.py`,或确认 `--db` 路径正确。 + +**Q: 数据库能跨机器用吗?** +A: 能。`dump_index.db` 是纯 SQLite 文件,拷贝即可,无需重装依赖。 diff --git a/build_index.py b/build_index.py new file mode 100644 index 0000000..df3c8ed --- /dev/null +++ b/build_index.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- +""" +build_index.py — 把 Unispect dump.txt 流式解析成 SQLite 结构化索引。 + +用法: + python build_index.py [dump.txt路径] [输出db路径] + +特性: + - 流式读取,绝不把整个文件读入内存(文件 7MB / 15.8 万行也毫无压力) + - 表 types: id, kind, name, base_type, line_no, src_order + - 表 fields: id, type_id, offset_hex, offset_dec, is_static, is_const, + name, field_type, line_no + - FTS5 全文索引: types_fts (name/base_type), fields_fts (name/field_type) + - 枚举常量(C)与静态字段(S)独立标记 +""" +import io +import os +import re +import sqlite3 +import sys +import time + +SRC_DEFAULT = r"F:\desktop\Release\dump.txt" +DB_DEFAULT = r"F:\desktop\CFramework\dump_index.db" + +TYPE_HEAD_RE = re.compile( + r"^\[(Class|Interface|Enum|Struct)\]\s+(.+?)(?:\s*:\s*(.+))?$" +) +FIELD_RE = re.compile( + r"^\s+\[([0-9A-Fa-f]+)\](?:\[([SC])\])?\s+(.+?)\s*:\s*(.+)$" +) +# 兜底: 缩进但没匹配字段正则的行(异常行, 记录到 log 便于排查, 不影响索引) +INDENTED_RE = re.compile(r"^\s+\S") + +KIND_MAP = {"Class": "class", "Interface": "interface", "Enum": "enum", "Struct": "struct"} + + +def clean_name(name: str) -> str: + """去掉 Unispect 的 `-.` 内部类前缀与泛型参数尾巴, 得到可读名。""" + n = name.strip() + if n.startswith("-."): + n = n[2:] + # 去掉泛型 `...<...>` 的尖括号部分用于短名 + short = re.split(r"[<`]", n, maxsplit=1)[0] + return n, short + + +def build(src: str, db: str) -> None: + if os.path.exists(db): + os.remove(db) + + t0 = time.time() + con = sqlite3.connect(db) + con.execute("PRAGMA journal_mode=OFF") + con.execute("PRAGMA synchronous=OFF") + con.execute("PRAGMA temp_store=MEMORY") + cur = con.cursor() + + cur.executescript( + """ + CREATE TABLE types ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL, + name TEXT NOT NULL, -- 原始名 (含 -. / 泛型) + short TEXT NOT NULL, -- 可读短名 + base_type TEXT, + line_no INTEGER, + src_order INTEGER + ); + CREATE TABLE fields ( + id INTEGER PRIMARY KEY, + type_id INTEGER NOT NULL REFERENCES types(id), + offset_hex TEXT NOT NULL, + offset_dec INTEGER NOT NULL, + is_static INTEGER NOT NULL DEFAULT 0, + is_const INTEGER NOT NULL DEFAULT 0, + name TEXT NOT NULL, + field_type TEXT, + line_no INTEGER + ); + CREATE INDEX idx_fields_type ON fields(type_id); + CREATE INDEX idx_fields_name ON fields(name); + CREATE INDEX idx_types_name ON types(name); + CREATE INDEX idx_types_short ON types(short); + CREATE INDEX idx_types_base ON types(base_type); + + CREATE VIRTUAL TABLE types_fts USING fts5( + name, short, base_type, content='types', content_rowid='id' + ); + CREATE VIRTUAL TABLE fields_fts USING fts5( + name, field_type, content='fields', content_rowid='id' + ); + """ + ) + + type_id = 0 + field_id = 0 + current = None # (type_id, kind, name) + type_count = 0 + field_count = 0 + anomaly_log = [] + + with io.open(src, "r", encoding="utf-8", errors="replace", newline="\n") as f: + for lineno, raw in enumerate(f, 1): + line = raw.rstrip("\r\n") + t = line.strip() + if not t: + continue + m = TYPE_HEAD_RE.match(t) + if m: + kind_raw, name, base = m.group(1), m.group(2), m.group(3) + type_id += 1 + name_full, short = clean_name(name) + base_s = (base or "").strip() + cur.execute( + "INSERT INTO types (id, kind, name, short, base_type, line_no, src_order) " + "VALUES (?,?,?,?,?,?,?)", + (type_id, KIND_MAP[kind_raw], name_full, short, base_s, + lineno, type_id), + ) + cur.execute( + "INSERT INTO types_fts (rowid, name, short, base_type) VALUES (?,?,?,?)", + (type_id, name_full, short, base_s), + ) + current = type_id + type_count += 1 + continue + + fm = FIELD_RE.match(line) + if fm and current is not None: + off_hex = fm.group(1) + flag = fm.group(2) or "" + fname = fm.group(3).strip() + ftype = fm.group(4).strip() + field_id += 1 + cur.execute( + "INSERT INTO fields (id, type_id, offset_hex, offset_dec, is_static, " + "is_const, name, field_type, line_no) VALUES (?,?,?,?,?,?,?,?,?)", + (field_id, current, off_hex, int(off_hex, 16), + 1 if flag == "S" else 0, + 1 if flag == "C" else 0, + fname, ftype, lineno), + ) + cur.execute( + "INSERT INTO fields_fts (rowid, name, field_type) VALUES (?,?,?)", + (field_id, fname, ftype), + ) + field_count += 1 + continue + + # 异常行: 缩进但解析不了, 或类型头后跟的杂行 + if INDENTED_RE.match(line) and current is not None: + if len(anomaly_log) < 50: + anomaly_log.append(f"L{lineno}: {t[:80]!r}") + + # 重建 FTS 索引内容表一致性 + cur.execute("INSERT INTO types_fts(types_fts) VALUES('rebuild')") + cur.execute("INSERT INTO fields_fts(fields_fts) VALUES('rebuild')") + con.commit() + + elapsed = time.time() - t0 + print(f"types = {type_count}") + print(f"fields = {field_count}") + print(f"anomaly = {len(anomaly_log)}") + for a in anomaly_log: + print(" !", a) + print(f"elapsed = {elapsed:.2f}s") + print(f"db = {db} ({os.path.getsize(db) / 1024:.0f} KB)") + + # 内置快速校验 + print("\n-- sanity checks --") + for q, label in [ + ("SELECT kind, COUNT(*) FROM types GROUP BY kind", "types by kind"), + ("SELECT COUNT(*) FROM types WHERE base_type IS NULL OR base_type=''", "types w/o base"), + ("SELECT COUNT(*) FROM fields WHERE is_static=1", "static fields"), + ("SELECT COUNT(*) FROM fields WHERE is_const=1", "const fields"), + ]: + rows = cur.execute(q).fetchall() + print(f"{label}: {rows}") + + con.close() + + +if __name__ == "__main__": + src = sys.argv[1] if len(sys.argv) > 1 else SRC_DEFAULT + db = sys.argv[2] if len(sys.argv) > 2 else DB_DEFAULT + build(src, db) diff --git a/dump_index.db b/dump_index.db new file mode 100644 index 0000000..696a076 Binary files /dev/null and b/dump_index.db differ diff --git a/query_index.py b/query_index.py new file mode 100644 index 0000000..1a53445 --- /dev/null +++ b/query_index.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +""" +query_index.py — 查询 dump SQLite 索引。 + +用法: + python query_index.py stats + python query_index.py type <名称子串> # 按类型名搜索 (LIKE) + python query_index.py short <短名> # 按可读短名精确/模糊 + python query_index.py base <基类子串> # 按基类搜索 + python query_index.py field <字段名子串> # 全局搜字段名 + python query_index.py ftype <字段类型子串> # 按字段类型搜 (如 List<) + python query_index.py show # 展开一个类型的全部字段 + python query_index.py dump # 原始格式还原 (offset + 标记) + +选项: + --limit N 默认 20 + --db PATH 默认 dump_index.db +""" +import argparse +import os +import sqlite3 +import sys + +DB_DEFAULT = r"F:\desktop\CFramework\dump_index.db" + + +def connect(db): + if not os.path.exists(db): + sys.exit(f"[!] 索引不存在: {db}\n 先运行: python build_index.py") + return sqlite3.connect(db) + + +def cmd_stats(con): + for q, label in [ + ("SELECT COUNT(*) FROM types", "types"), + ("SELECT COUNT(*) FROM fields", "fields"), + ("SELECT kind, COUNT(*) FROM types GROUP BY kind", "types by kind"), + ]: + print(f"{label}: {con.execute(q).fetchall()}") + # 字段最多/最少的前几个类型 + print("\ntop 10 types by field count:") + rows = con.execute( + """SELECT t.id, t.kind, t.name, COUNT(f.id) c FROM types t + LEFT JOIN fields f ON f.type_id = t.id + GROUP BY t.id ORDER BY c DESC LIMIT 10""" + ).fetchall() + for r in rows: + print(f" [{r[0]:>5}] {r[1]:<9} {r[2][:60]:<60} fields={r[3]}") + + +def cmd_types(con, needle, limit): + like = f"%{needle}%" + rows = con.execute( + """SELECT id, kind, name, base_type, + (SELECT COUNT(*) FROM fields f WHERE f.type_id=t.id) AS fc + FROM types t WHERE name LIKE ? OR short LIKE ? + ORDER BY fc DESC LIMIT ?""", + (like, like, limit), + ).fetchall() + print(f"{len(rows)} match(es) for type name '{needle}':") + for r in rows: + print(f" [{r[0]:>5}] {r[1]:<9} {r[2][:70]:<70} base={r[3] or '-'} fields={r[4]}") + + +def cmd_base(con, needle, limit): + rows = con.execute( + """SELECT id, kind, name, base_type, + (SELECT COUNT(*) FROM fields f WHERE f.type_id=t.id) AS fc + FROM types t WHERE base_type LIKE ? ORDER BY fc DESC LIMIT ?""", + (f"%{needle}%", limit), + ).fetchall() + print(f"{len(rows)} type(s) with base like '{needle}':") + for r in rows: + print(f" [{r[0]:>5}] {r[1]:<9} {r[2][:70]:<70} base={r[3] or '-'} fields={r[4]}") + + +def cmd_fields(con, needle, limit): + rows = con.execute( + """SELECT f.id, t.kind, t.name, f.offset_hex, f.name, f.field_type, + f.is_static, f.is_const + FROM fields f JOIN types t ON t.id = f.type_id + WHERE f.name LIKE ? ORDER BY f.name LIMIT ?""", + (f"%{needle}%", limit), + ).fetchall() + print(f"{len(rows)} field(s) named like '{needle}':") + for r in rows: + flags = ("S" if r[6] else "") + ("C" if r[7] else "") + print(f" [{r[0]:>6}] {r[1]:<9} {r[2][:48]:<48} [+{r[3]:>4}]{flags:<3} {r[4]} : {r[5]}") + + +def cmd_ftype(con, needle, limit): + rows = con.execute( + """SELECT f.id, t.kind, t.name, f.offset_hex, f.name, f.field_type + FROM fields f JOIN types t ON t.id = f.type_id + WHERE f.field_type LIKE ? ORDER BY f.name LIMIT ?""", + (f"%{needle}%", limit), + ).fetchall() + print(f"{len(rows)} field(s) with type like '{needle}':") + for r in rows: + print(f" [{r[0]:>6}] {r[1]:<9} {r[2][:48]:<48} [+{r[3]:>4}] {r[4]} : {r[5]}") + + +def resolve_type(con, arg): + """arg 可以是数字 id 或类型名子串; 返回唯一类型或 None""" + if arg.isdigit(): + row = con.execute( + "SELECT id, kind, name, base_type, line_no FROM types WHERE id=?", + (int(arg),), + ).fetchone() + if row: + return row + rows = con.execute( + """SELECT id, kind, name, base_type, line_no FROM types + WHERE name = ? OR short = ? LIMIT 5""", + (arg, arg), + ).fetchall() + if len(rows) == 1: + return rows[0] + if len(rows) > 1: + print(f"[!] '{arg}' 有 {len(rows)} 个匹配, 用 id 精确定位:") + for r in rows: + print(f" [{r[0]}] {r[1]} {r[2]}") + return None + # 子串兜底 + rows = con.execute( + "SELECT id, kind, name, base_type, line_no FROM types WHERE name LIKE ? LIMIT 10", + (f"%{arg}%",), + ).fetchall() + if len(rows) == 1: + return rows[0] + if len(rows) > 1: + print(f"[!] '{arg}' 有 {len(rows)} 个匹配:") + for r in rows[:10]: + print(f" [{r[0]}] {r[1]} {r[2]}") + return None + + +def cmd_show(con, arg): + t = resolve_type(con, arg) + if not t: + sys.exit(f"[!] 未找到类型: {arg}") + tid, kind, name, base, line = t + print(f"== [{tid}] {kind} {name} : {base or '-'} (line {line}) ==") + rows = con.execute( + """SELECT offset_hex, is_static, is_const, name, field_type + FROM fields WHERE type_id=? ORDER BY offset_dec, id""", + (tid,), + ).fetchall() + if not rows: + print(" (无字段)") + for off, st, cs, fname, ftype in rows: + flags = ("[S]" if st else "") + ("[C]" if cs else "") + print(f" +{off:<5} {flags:<4} {fname} : {ftype}") + print(f" -- {len(rows)} fields --") + + +def cmd_dump(con, arg): + """按 dump.txt 原始格式还原类型""" + t = resolve_type(con, arg) + if not t: + sys.exit(f"[!] 未找到类型: {arg}") + tid, kind, name, base, line = t + print(f"[{kind.title()}] {name} : {base or ''}") + for off, st, cs, fname, ftype in con.execute( + """SELECT offset_hex, is_static, is_const, name, field_type + FROM fields WHERE type_id=? ORDER BY offset_dec, id""", + (tid,), + ): + flags = ("[S]" if st else "") + ("[C]" if cs else "") + print(f" [{off}]{flags} {fname} : {ftype}") + + +def main(): + ap = argparse.ArgumentParser(description="dump.txt SQLite 索引查询") + ap.add_argument("cmd", choices=["stats", "type", "short", "base", "field", "ftype", "show", "dump"]) + ap.add_argument("needle", nargs="?", default="") + ap.add_argument("--limit", type=int, default=20) + ap.add_argument("--db", default=DB_DEFAULT) + args = ap.parse_args() + + con = connect(args.db) + if args.cmd == "stats": + cmd_stats(con) + elif args.cmd == "type": + cmd_types(con, args.needle, args.limit) + elif args.cmd == "short": + cmd_types(con, args.needle, args.limit) # short 也走 name+short LIKE + elif args.cmd == "base": + cmd_base(con, args.needle, args.limit) + elif args.cmd == "field": + cmd_fields(con, args.needle, args.limit) + elif args.cmd == "ftype": + cmd_ftype(con, args.needle, args.limit) + elif args.cmd == "show": + cmd_show(con, args.needle) + elif args.cmd == "dump": + cmd_dump(con, args.needle) + con.close() + + +if __name__ == "__main__": + main()