refactor(结构): 分层重组 + 清除全部废弃路线

做了什么:
- 包改为分层: tcaptcha/{protocol,telemetry,solvers,ocr,adapters,models,tests}
  * protocol/  client.py  transport.py  proxy.py
  * telemetry/ crypto.py  values.py  key.py  opcodes.py
               disasm.py  program.py  layout.py  modules.py
  * solvers/   text_click.py  icon_click.py  icon_click_rotate.py  slider.py
  * ocr/       engines.py  slicer.py
  * adapters/  one_ms.py
- tools/ 里原本是**包运行时依赖**的 tdc_{disasm,program,layout,modules}.py 并入 telemetry/
  (放在 tools/ 属于分层倒置),tools/ 只留 CLI: tdc_key_pure.py / tdc_profile.py / cdp_*.js
- crypto.py 529 -> 340 行: 切除「成功样本模板回填」与「Node 插桩兜底」两条老路;
  generate_telemetry 只剩 python_values 一种模式,传别的会直接报错并说明原因
- 求解器改名分清: solver.py -> solvers/icon_click.py(client 实际在用的 YOLO+ResNet18 ONNX),
  icon_click_solver.py -> solvers/icon_click_rotate.py(仅测试在用,留作对照)
- pyproject: 去掉 ultralytics/onnx(训练栈已删),补上真正在用的 onnxruntime

删除了什么:
- 雨云整条线 + dist/rainyun_qinglong(104M 旧快照)
- 模板回填: models/success_collect_template.b64、tdc_worker.js、analyze_collect_layout.py
- 「补环境」时代浏览器 JS: chrome_fetch/manual_captcha/probe_captcha/solve_in_browser/tdc_browser
- 旧 VM 探索工具 10 个(tdc_collectgen/tdc_slotmap/tdc_lift/tdc_vm_run/tdc_query/tdc_probe 等)
  + 8 个基于它们的旧 verify 脚本
- 训练/标注/评估: train_classifier/train_detector/prepare_yolo/web_labeler/collector/
  siamese/evaluate/benchmark_verify
- tcaptcha/core_js/(60 个分析产物)、TOOLCHAIN_REPORT.md(整篇在描述已删工具)
- 3 个已无任何引用的 PyTorch 权重:icon_classifier.pt 43M / yolo_detector.pt 5.9M /
  siamese_matcher.pt 1.6M —— 线上只跑 onnx,tcaptcha/ 117M -> 66M

为什么:
- 项目后面要专门承载「滑块 / 文字点选 / 图标点选」三种求解器;扁平包 + 运行时依赖混在
  tools/ + 两条已废弃的 collect 生成路线,会让后续扩展持续踩坑

怎么验证:
- .venv/bin/python -m tcaptcha.selftest                    全绿
- scripts/verify/verify_offline.py                         逐槽全等 + 分段几何一致
- scripts/verify/verify_values.py                          4 份真机明文 152/152 逐槽全等
- scripts/verify/send_test.py --mode python_values         1ms 签到页文字点选 errorCode=0
- 回滚点: tag backup/pre-refactor-20261012
This commit is contained in:
yml2213
2026-10-12 12:34:57 +08:00
parent 51dff870cf
commit 344eea2194
169 changed files with 599 additions and 83331 deletions
-1
View File
@@ -1 +0,0 @@
{"seq": [8, 9, 10, 15, 18, 19, 20, 21, 27, 28, 31, 33, 34, 36, 37, 38, 39, 40, 41, 26, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 55, 1, 56, 57, 58], "kinds": [2, 1, null, null, null, null, null, null, 1, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 1, null, null, null, null, null, 1, null, 1, null], "v0s": ["vXc29RxhvT34Qjg4BO81Rcg6kcS36LwS", "1", "1440", "0", "0", "0", "2560", "98k", "[]", "+08", "https://turing.captcha.gtimg.com/1/template/drag_ele.html?rand=1519713624347", "1440-2560-2530-24-*-*-|-*", "0", "MacIntel", "2", "https://1ms.run/console/personal/benefits/check-in", "1791736469", "", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36", "1984571394", "ANGLE (Apple, ANGLE Metal Renderer: Apple M4, Unspecified Version)", "iframe", "", "0", "1023", "Google Inc. (Apple)", "", "[360,360]", "1791736491", "24", "UTF-8", "10", "GgoAAAANSUhEUgAAASwAAACWCAYAAATf3v8bh+ryWcfSBSLkmN3N1PQinPL7Bi3TG0qE2CogAAAABJRU5ErkJggg==", "641547489,1791732001", "2", "[\"zh-CN\",\"zh\"]", "0"], "sample_cd": [1791736322, 2560, [[1, 1, 12]], 0, "1440-2560-2530-24-*-*-|-*", 1440, "iframe", 0, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36", "ANGLE (Apple, ANGLE Metal Renderer: Apple M4, Unspecified Version)", "https://turing.captcha.gtimg.com/1/template/drag_ele.html?rand=1519713624347", "", 0, "+08", 1023, "", [360, 360], "UTF-8", "", 0, 2, 1791736260, 2086701610, 3, 1, "https://1ms.run/console/personal/benefits/check-in", "98k", [], "MacIntel", "Google Inc. (Apple)", "GgoAAAANSUhEUgAAASwAAACWCAYAAATf3v8bh+ryWcfSBSLkmN3N1PQinPL7Bi3TG0qE2CogAAAABJRU5ErkJggg==", 0, 10, 641547489, 1791732001, 24, ["zh-CN", "zh"], 0]}
-180
View File
@@ -1,180 +0,0 @@
# -*- coding: utf-8 -*-
"""
按构建生成 collect 的回归测试(承 PROGRESS.md #34/#35/#36;AGENTS.md §5.9)。
固化的性质:
1. `outer_commas`: 只数**括号外**的逗号 —— 方括号内的逗号不构成顶层分隔
('[]' / '[360,360]' / '["zh-CN","zh"]' 拼回去仍是一个值);
2. `SlotSpec` 推算出的槽数 == 真机明文 cd 的实际槽数;
3. 重排是**置换**:值集合不变(排序后逐项相同);
4. 加密/解密**回环一致**(明文逐字节还原);
5. 槽数不匹配的样本**必须报错**,不能静默错位。
样本:
* tcaptcha/tests/fixtures/tdc_slotmap_case.json —— 合成用例(已知正确答案)
* har/online/<时间戳>/tdc_*.js —— **在线抓取的全新构建**(真机数据)
运行:
.venv/bin/python -m tcaptcha.tests.test_collectgen
"""
from __future__ import annotations
import base64
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
if str(ROOT / "tools") not in sys.path:
sys.path.insert(0, str(ROOT / "tools"))
from tcaptcha.crypto import xtea_decrypt_bytes # noqa: E402
import tdc_collectgen as cg # noqa: E402
FIX = ROOT / "tcaptcha" / "tests" / "fixtures" / "tdc_slotmap_case.json"
# 固定密钥,仅用于回环自洽(不代表任何真实构建)
KEY = [1497711441, 1213874789, 1799903316, 1835232858]
_fails: list = []
def check(name: str, cond: bool, detail: str = "") -> None:
print(f" [{'PASS' if cond else 'FAIL'}] {name}" + (f" {detail}" if detail else ""))
if not cond:
_fails.append(name)
def main() -> int:
if not FIX.exists():
print(f"缺少 fixture: {FIX}")
return 1
case = json.loads(FIX.read_text(encoding="utf-8"))
seq, kinds, v0s, sample_cd = case["seq"], case["kinds"], case["v0s"], case["sample_cd"]
print(f"fixture: 项 {len(seq)} / kind {len(kinds)} / v0 {len(v0s)} / 真机 cd {len(sample_cd)}")
print("\n[1] outer_commas: 只数括号外的逗号")
check("'[]' -> 0", cg.outer_commas("[]") == 0)
check("'[360,360]' -> 0", cg.outer_commas("[360,360]") == 0)
check("'[\"zh-CN\",\"zh\"]' -> 0", cg.outer_commas('["zh-CN","zh"]') == 0)
check("'641547489,1791732001' -> 1", cg.outer_commas("641547489,1791732001") == 1)
check("'a,b,c' -> 2", cg.outer_commas("a,b,c") == 2)
print("\n[2] 槽数推算 == 真机明文槽数")
spec = cg.SlotSpec(seq, kinds, v0s)
print(f" {spec.describe()}")
check("槽数一致", spec.n_slots == len(sample_cd),
f"{spec.n_slots} vs {len(sample_cd)}")
check("fixture 覆盖 kind=1", any(k == 1 for k in kinds))
check("fixture 覆盖 kind=2", any(k == 2 for k in kinds))
print("\n[3] 重排是置换(值集合不变)")
out = cg.reorder(sample_cd, spec)
key = lambda v: json.dumps(v, sort_keys=True, ensure_ascii=False)
check("长度不变", len(out) == len(sample_cd), f"{len(out)} vs {len(sample_cd)}")
check("值集合相同(排序后逐项相等)",
sorted(map(key, out)) == sorted(map(key, sample_cd)))
print("\n[4] 加密/解密回环")
plain = cg.build_plain(out)
b64 = cg.encrypt_plain(plain, KEY)
back = xtea_decrypt_bytes(base64.b64decode(b64), KEY).decode("latin1").rstrip("\x00").strip()
check("明文逐字节还原", back == plain)
back_obj = json.loads(back)
check("cd 还原", back_obj["cd"] == out)
check("sd 还原", back_obj["sd"] == {"od": "C", "ft": "qf_7Pf__H"})
print("\n[5] 槽数不匹配必须报错")
raised = False
try:
cg.reorder(sample_cd[:-1], spec) # src=None: 契约是"同序"
except ValueError:
raised = True
check("少一槽 -> ValueError", raised)
# ---------------------------------------------------------------- #
# [6] 强断言:重排**必须真的重排**(旧实现的 bug 就是被"是置换"放过的)
# ---------------------------------------------------------------- #
print("\n[6] reorder 不是恒等映射(抓旧 bug)")
print(" 旧实现的循环是 sample[slot:slot+n],其中 slot 是**目标**槽号,")
print(" 两份构建顺序不同时全盘错位;而'值集合不变'这个弱断言抓不住它。")
# 把清单顺序打乱,构造"另一份构建"
import random
idx = list(range(len(seq)))
random.Random(11).shuffle(idx)
seq2 = [seq[i] for i in idx]
kinds2 = [kinds[i] for i in idx]
v0s2 = [v0s[i] for i in idx]
# 给每项一个唯一语义名(真实场景由 tdc_modules 的指纹提供)
names1 = ["n%d" % i for i in range(len(seq))]
names2 = ["n%d" % i for i in idx]
src = cg.spec_from_seq(seq, kinds, names1, sample_cd=sample_cd, v0s=v0s)
dst = cg.spec_from_seq(seq2, kinds2, names2, v0s=v0s2)
# 正确答案:按 src 的 assign 把每项的值搬到目标顺序
truth: list = []
for i in idx:
_i, _m, sl, nn = src.assign[i]
truth.extend(sample_cd[sl:sl + nn])
try:
out2 = cg.reorder(sample_cd, dst, src)
check("重排结果 == 已知正确答案", out2 == truth)
check("重排结果 != 原样(证明真的重排了)", out2 != list(sample_cd))
except cg.UnmappedSlotsError as exc:
check("重排结果 == 已知正确答案", False, f"抛错: {str(exc)[:80]}")
# ---------------------------------------------------------------- #
# [7] 用在线抓取的**全新构建**做端到端重排(真机数据,非合成)
# ---------------------------------------------------------------- #
online = sorted((ROOT / "har" / "online").glob("*/tdc_*.js")) if (
ROOT / "har" / "online").exists() else []
print("\n[7] 在线新构建 + 语义名锚点的端到端重排")
if len(online) < 2:
print(" (跳过: 需要 ≥2 份 har/online/*/tdc_*.js;"
"用 scripts/fetch_online_samples.py --n 4 抓取)")
else:
try:
import tdc_modules
except Exception as exc:
check("导入 tdc_modules", False, str(exc)[:80])
online = []
if len(online) >= 2:
def names_of(f):
return [r["name"] for r in tdc_modules.analyze(f)]
na = names_of(online[0])
n = len(na)
check("新构建清单 37 项", n == 37, "实际 %d" % n)
check("语义名全部唯一(37/37)", len(set(na)) == n,
"唯一 %d/%d" % (len(set(na)), n))
check("无 unknown", not any(x == "unknown" for x in na))
# 给每个语义名一个假值,按 A 顺序排成 cd_A,重排到 B 顺序
fake = {nm: "V<%s>" % nm for nm in na}
cd_a = [fake[nm] for nm in na]
seqa = [r["module"] for r in tdc_modules.analyze(online[0])]
srca = cg.spec_from_seq(seqa, [None] * n, na, sample_cd=cd_a)
all_ok = True
for f in online[1:]:
nb = names_of(f)
seqb = [r["module"] for r in tdc_modules.analyze(f)]
dstb = cg.spec_from_seq(seqb, [None] * len(nb), nb)
truthb = [fake[nm] for nm in nb]
try:
got = cg.reorder(cd_a, dstb, srca)
good = got == truthb and got != cd_a
except Exception:
good = False
all_ok &= good
print(" %-28s 重排正确=%s" % (f.parent.name + "/" + f.name[:12], good))
check("全部新构建重排正确且非恒等", all_ok)
print(f"\n{'=' * 62}")
if _fails:
print(f"❌ collectgen 测试失败: {_fails}")
return 1
print("✅ collectgen 测试全部通过")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+4 -4
View File
@@ -26,7 +26,7 @@ if str(ROOT) not in sys.path:
if str(ROOT / "tools") not in sys.path:
sys.path.insert(0, str(ROOT / "tools"))
from tcaptcha.crypto import (build_collect, selfcheck, split_collect_segments, # noqa: E402
from tcaptcha.telemetry.crypto import (build_collect, selfcheck, split_collect_segments, # noqa: E402
xtea_decrypt_bytes)
# 真机同会话 collect 的**解密明文**(2026-10-12,1ms aid=195672344 的验证码 iframe)。
@@ -104,9 +104,9 @@ def test_build_collect_roundtrip_single_space_pad() -> None:
def test_offline_generate_and_roundtrip() -> None:
"""纯离线:tdc.js -> 37 语义名 -> 38 槽 -> 4 段明文 -> 加密 -> 解回逐槽相等。"""
import tdc_modules as tm
from tcaptcha.tdc_key import extract_xtea_key_pure
from tcaptcha.tdc_values import Session, build_plain, generate_cd, trajectory_index
from tcaptcha.telemetry import modules as tm
from tcaptcha.telemetry.key import extract_xtea_key_pure
from tcaptcha.telemetry.values import Session, build_plain, generate_cd, trajectory_index
build = ROOT / "tcaptcha" / "tdc_sample.js"
js = build.read_text(encoding="utf-8", errors="replace")
+1 -1
View File
@@ -25,7 +25,7 @@ ROOT = Path(__file__).resolve().parent.parent.parent
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from tcaptcha.glyph_solver import TextClickSolver, parse_instruction # noqa: E402
from tcaptcha.solvers.text_click import TextClickSolver, parse_instruction # noqa: E402
TOLERANCE = 22 # 像素容差 (汉字包围盒中心 vs 人工标注中心)
+1 -1
View File
@@ -25,7 +25,7 @@ ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from tcaptcha.icon_click_solver import IconClickSolver # noqa: E402
from tcaptcha.solvers.icon_click_rotate import IconClickSolver # noqa: E402
FIX = ROOT / "tcaptcha" / "tests" / "fixtures"
# 期望点击顺序: ("0" 右下) -> (倾斜的图表图标 左上) -> ("7" 中上),坐标是 672x480 原图口径
+1 -1
View File
@@ -33,7 +33,7 @@ ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from tcaptcha.slider_solver import SliderSolver, piece_mask, sprite_region # noqa: E402
from tcaptcha.solvers.slider import SliderSolver, piece_mask, sprite_region # noqa: E402
FIX = ROOT / "tcaptcha" / "tests" / "fixtures"
+5 -6
View File
@@ -4,9 +4,9 @@
固化的性质:
1. **跨构建通用**: opcode 编号每份构建整体重排, 但仍能按构建推导出正确语义表;
2. **纯 Python == Node 神谕**: 6 份构建(含 1 份新抓的实时 tdc.js)的期望密钥
2. **纯 Python == Node 神谕**: 5 份构建(含 1 份新抓的实时 tdc.js)的期望密钥
全部由 Node `extract_xtea_key` 逐字对照确认过, 此处固化为断言 (不再依赖 Node 跑测试);
3. **定位不写死 pc**: 断言覆盖 core_js / root / 1ms(旧+两份实时) / 实时新抓 共 6 份
3. **定位不写死 pc**: 断言覆盖 core_js / root / 1ms(旧+两份实时) / 实时新抓 共 5 份
不同构建, 任一份 opcode 重排或寄存器分配变化都会立刻暴露。
运行:
@@ -21,15 +21,14 @@ ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from tcaptcha.tdc_key import extract_xtea_key_pure # noqa: E402
from tcaptcha.tdc_opcodes import derive_opcode_table # noqa: E402
from tcaptcha.telemetry.key import extract_xtea_key_pure # noqa: E402
from tcaptcha.telemetry.opcodes import derive_opcode_table # noqa: E402
# (相对路径, 期望有效密钥, 是否必须存在)
# 期望值来自 Node 插桩 extract_xtea_key, 逐字一致。
# 入库样本 (core_js / root / 实时新抓 fixture) 为必需门禁; har/** 被 .gitignore 忽略,
# 入库样本 (root / 实时新抓 fixture) 为必需门禁; har/** 被 .gitignore 忽略,
# 本地有这些样本时会一并回归, 缺少时自动跳过。
FIXTURES = [
("tcaptcha/core_js/tdc_sample.js", [1818510928, 1381395306, 1245924197, 1768384072], True),
("tcaptcha/tdc_sample.js", [1448563793, 1850559083, 1733513813, 1364613714], True),
# 2026-10-11 现场从 turing.captcha.qcloud.com 下发的 tdc.js (入库固化, 防"仅本地可验")
("tcaptcha/tests/fixtures/tdc_fresh_2026-10-11.js",
-128
View File
@@ -1,128 +0,0 @@
# -*- coding: utf-8 -*-
"""
XTEA 算法还原 + tdc.js 密钥提取 回归测试 (AGENTS.md §5.9)
固化的性质:
1. 标准 XTEA 加解密往返
2. 与真实 tdc.js **逐字节一致** (算法还原正确的判据, §3.0)
3. 插桩能从 tdc.js 复原密钥, 且密钥按构建固定 (§5.5 受控环境逐字节一致)
4. 模板标定 + 纯 Python 回填 的产物可被同一把密钥解回预期结构
运行:
.venv/bin/python -m tcaptcha.tests.test_xtea_key
"""
from __future__ import annotations
import base64
import json
import struct
import subprocess
import sys
import urllib.parse
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from tcaptcha.crypto import ( # noqa: E402
DEFAULT_KEY,
generate_collect_from_template,
recover_payload_template,
xtea_decrypt_block,
xtea_decrypt_bytes,
xtea_encrypt_bytes,
extract_xtea_key,
)
ROOT = Path(__file__).resolve().parents[2]
SAMPLE = ROOT / "tcaptcha" / "tdc_sample.js"
SAMPLE_KEY = [1448563793, 1850559083, 1733513813, 1364613714] # 等价于 DEFAULT_KEY + 两个加盐
CD_PRE = b'{"cd":[[],"1920-1080-1055-24-*-*-|-*",0,0,"", ' # 旧版明文首段 (已补齐 48B)
_fails: list[str] = []
def check(name: str, cond: bool, detail: str = "") -> None:
print(f" [{'PASS' if cond else 'FAIL'}] {name}" + (f" {detail}" if detail else ""))
if not cond:
_fails.append(name)
def run_tdc(script: str, points, options) -> dict:
proc = subprocess.run(
["node", str(ROOT / "tcaptcha" / "tdc_worker.js"), "-", json.dumps(points),
json.dumps(options)],
input=script, capture_output=True, text=True, timeout=180)
return json.loads(proc.stdout.strip().splitlines()[-1])
def test_xtea_roundtrip() -> None:
print("1) 标准 XTEA 往返")
key = list(SAMPLE_KEY)
for chunk in (b"hello!!", bytes(range(32)), b"x" * 8):
enc = xtea_encrypt_bytes(chunk, key)
dec = xtea_decrypt_bytes(enc, key)
check(f"往返 {len(chunk)}B", dec.startswith(chunk), f"{chunk[:8]!r}")
# 已知明文 -> 与 tdc.js 密文逐字节一致
enc = xtea_encrypt_bytes(CD_PRE, key)
check("加密结果可被同密钥解回", xtea_decrypt_bytes(enc, key).rstrip(b"\x00") == CD_PRE.rstrip(b"\x00"))
def test_byte_exact_vs_tdcjs() -> None:
print("2) 纯 Python 密文 == 真实 tdc.js 密文 (逐字节)")
script = SAMPLE.read_text(encoding="utf-8", errors="replace")
job = run_tdc(script, [[100, 100], [200, 200], [300, 300]], {"aid": "195672344"})
real = base64.b64decode(urllib.parse.unquote(job["collect"]))
key = extract_xtea_key(script)
check("提取到的密钥 == 已知 K'", key == SAMPLE_KEY, str(key))
# 旧版首段是常量 cd_pre, 可逐字节对拍
check("前 48 字节逐字节一致",
xtea_encrypt_bytes(CD_PRE, key) == real[:48],
xtea_encrypt_bytes(CD_PRE, key)[:8].hex())
def test_key_fixed_per_build() -> None:
print("3) 密钥按构建固定 (同一脚本多次运行首块一致)")
script = SAMPLE.read_text(encoding="utf-8", errors="replace")
heads = set()
for _ in range(2):
job = run_tdc(script, [[10, 10], [20, 20], [30, 30]], {"aid": "195672344"})
heads.add(base64.b64decode(urllib.parse.unquote(job["collect"]))[:8])
check("两次运行首块相同", len(heads) == 1, heads.pop().hex())
def test_template_refill() -> None:
print("4) 模板标定 + 纯 Python 回填")
script = SAMPLE.read_text(encoding="utf-8", errors="replace")
template, key = recover_payload_template(script)
check("模板以 {\"cd\":[ 开头", template.startswith('{"cd":['), template[:24])
try:
json.loads(template)
check("模板是合法 JSON", True)
except Exception as exc: # noqa: BLE001
check("模板是合法 JSON", False, str(exc))
collect = generate_collect_from_template(template, [(449, 254), (48, 53), (148, 252)], key)
plain = xtea_decrypt_bytes(base64.b64decode(urllib.parse.unquote(collect)), key) \
.decode("latin1").rstrip(" ")
check("回填产物可解回 {\"cd\":[ 开头", plain.startswith('{"cd":['), plain[:24])
check("回填产物以 }} 结尾", plain.rstrip().endswith("}}"))
check("轨迹已换成 3 次点击的规模", len(plain) != len(template),
f"{len(template)} -> {len(plain)}")
def main() -> int:
if not SAMPLE.exists():
print(f"缺少样本 {SAMPLE}")
return 1
for fn in (test_xtea_roundtrip, test_byte_exact_vs_tdcjs,
test_key_fixed_per_build, test_template_refill):
fn()
print()
if _fails:
print(f"❌ XTEA/密钥 回归失败 {len(_fails)} 项: {_fails}")
return 1
print("✅ XTEA/密钥 回归全部通过")
return 0
if __name__ == "__main__":
raise SystemExit(main())