升级半自动工作台

2026-09-11 22:56:03 +08:00
parent f1c664f622
commit 4d6cded0b0
13 changed files with 3269 additions and 231 deletions
@@ -0,0 +1,147 @@
+"""
+腾讯天御点选验证码求解器端到端评估脚本
+评估指标:
+1. 单点命中率 (Hit Rate per Point, distance <= 30px)
+2. 无序3点命中率 (All 3 icons detected regardless of order)
+3. 严格时序三点命中率 (Full 3-point Sequence Accuracy: P1->T1, P2->T2, P3->T3)
+"""
+
+import json
+import logging
+import sys
+from pathlib import Path
+
+# 确保项目根目录在 sys.path 中
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+import numpy as np
+from tcaptcha.solver import CaptchaSolver
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+logger = logging.getLogger("evaluate")
+
+
+def evaluate_solver(labels_path: Path = Path("tcaptcha/data/labels.jsonl"), tolerance_px: float = 35.0):
+    if not labels_path.exists():
+        logger.error(f"Labels file not found: {labels_path}")
+        return
+
+    solver = CaptchaSolver()
+
+    samples = []
+    with open(labels_path, "r", encoding="utf-8") as f:
+        for line in f:
+            if line.strip():
+                samples.append(json.loads(line.strip()))
+
+    total_samples = len(samples)
+    total_points = total_samples * 3
+
+    exact_seq_success = 0
+    any_order_success = 0
+    total_points_hit = 0
+    point_errors = []  # distances for hits
+
+    failed_samples = []
+
+    print(f"\n================ 开始评估 (共 {total_samples} 个样本) ================\n")
+
+    for idx, s in enumerate(samples, 1):
+        sid = s["sample_id"]
+        gt_points = [(p["x"], p["y"]) for p in s["points"]]
+        bg = Path(f"tcaptcha/data/raw/{sid}_bg.png")
+        prompt = Path(f"tcaptcha/data/raw/{sid}_prompt.png")
+
+        if not bg.exists() or not prompt.exists():
+            logger.warning(f"[{idx}/{total_samples}] {sid} 缺失图像文件")
+            continue
+
+        pred_points = solver.solve(bg, prompt)
+        if len(pred_points) < 3:
+            logger.warning(f"[{idx}/{total_samples}] {sid} 预测点数不足3个: {pred_points}")
+            failed_samples.append((sid, "insufficient_points", gt_points, pred_points))
+            continue
+
+        # 1. 严格顺序匹配评估
+        seq_hit = True
+        sample_point_hits = 0
+        distances = []
+        for i in range(3):
+            gx, gy = gt_points[i]
+            px, py = pred_points[i]
+            dist = np.hypot(px - gx, py - gy)
+            distances.append(dist)
+            if dist <= tolerance_px:
+                sample_point_hits += 1
+                point_errors.append(dist)
+            else:
+                seq_hit = False
+
+        total_points_hit += sample_point_hits
+
+        # 2. 无序匹配评估 (检查是否3个图标都找出来了，哪怕顺序搞错)
+        # 为每个预测点找最近的GT点
+        used_gt = set()
+        for px, py in pred_points:
+            best_gt = None
+            min_d = 9999
+            for gi, (gx, gy) in enumerate(gt_points):
+                if gi in used_gt:
+                    continue
+                d = np.hypot(px - gx, py - gy)
+                if d < min_d:
+                    min_d = d
+                    best_gt = gi
+            if best_gt is not None and min_d <= tolerance_px:
+                used_gt.add(best_gt)
+
+        all_detected = (len(used_gt) == 3)
+        if all_detected:
+            any_order_success += 1
+
+        if seq_hit:
+            exact_seq_success += 1
+            status = "✅ PASS"
+        else:
+            status = "❌ FAIL"
+            failed_samples.append((sid, f"dist={[round(d,1) for d in distances]}", gt_points, pred_points))
+
+        print(f"[{idx:02d}/{total_samples}] {sid}: {status} | 点命中: {sample_point_hits}/3 | 距离: {[round(d, 1) for d in distances]}")
+
+    # 统计总结
+    seq_acc = exact_seq_success / total_samples * 100
+    order_agnostic_acc = any_order_success / total_samples * 100
+    pt_hit_rate = total_points_hit / total_points * 100
+    avg_err = np.mean(point_errors) if point_errors else 0.0
+
+    print("\n" + "=" * 60)
+    print("🎯 评估结果报告 (Evaluation Report):")
+    print(f"  - 样本总数 (Total Samples): {total_samples}")
+    print(f"  - 判定容差半径 (Tolerance): <= {tolerance_px} px")
+    print(f"  - 单点命中率 (Point Hit Rate): {total_points_hit}/{total_points} ({pt_hit_rate:.2f}%)")
+    print(f"  - 无序召回率 (Order-agnostic 3-Point Hit Rate): {any_order_success}/{total_samples} ({order_agnostic_acc:.2f}%)")
+    print(f"  - 🌟 严格时序三点识别率 (Sequence Accuracy): {exact_seq_success}/{total_samples} ({seq_acc:.2f}%)")
+    print(f"  - 命中点平均中心偏差 (Avg Center Offset): {avg_err:.2f} px")
+    print("=" * 60)
+
+    if failed_samples:
+        print(f"\n❌ 失败样本分析 (共 {len(failed_samples)} 个):")
+        for sid, reason, gt, pred in failed_samples[:10]:
+            print(f"  - {sid}: {reason} | GT={gt} vs Pred={pred}")
+        if len(failed_samples) > 10:
+            print(f"  ... 还有 {len(failed_samples) - 10} 个失败样本未列出")
+
+    return {
+        "seq_acc": seq_acc,
+        "order_agnostic_acc": order_agnostic_acc,
+        "pt_hit_rate": pt_hit_rate,
+        "failed_count": len(failed_samples)
+    }
+
+
+if __name__ == "__main__":
+    import sys
+    tol = float(sys.argv[1]) if len(sys.argv) > 1 else 35.0
+    evaluate_solver(tolerance_px=tol)