148 lines
5.2 KiB
Python
148 lines
5.2 KiB
Python
"""
|
|
腾讯天御点选验证码求解器端到端评估脚本
|
|
评估指标:
|
|
1. 单点命中率 (Hit Rate per Point, distance <= 30px)
|
|
2. 无序3点命中率 (All 3 icons detected regardless of order)
|
|
3. 严格时序三点命中率 (Full 3-point Sequence Accuracy: P1->T1, P2->T2, P3->T3)
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# 确保项目根目录在 sys.path 中
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
if str(PROJECT_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
import numpy as np
|
|
from tcaptcha.solver import CaptchaSolver
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
|
logger = logging.getLogger("evaluate")
|
|
|
|
|
|
def evaluate_solver(labels_path: Path = Path("tcaptcha/data/labels.jsonl"), tolerance_px: float = 35.0):
|
|
if not labels_path.exists():
|
|
logger.error(f"Labels file not found: {labels_path}")
|
|
return
|
|
|
|
solver = CaptchaSolver()
|
|
|
|
samples = []
|
|
with open(labels_path, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
if line.strip():
|
|
samples.append(json.loads(line.strip()))
|
|
|
|
total_samples = len(samples)
|
|
total_points = total_samples * 3
|
|
|
|
exact_seq_success = 0
|
|
any_order_success = 0
|
|
total_points_hit = 0
|
|
point_errors = [] # distances for hits
|
|
|
|
failed_samples = []
|
|
|
|
print(f"\n================ 开始评估 (共 {total_samples} 个样本) ================\n")
|
|
|
|
for idx, s in enumerate(samples, 1):
|
|
sid = s["sample_id"]
|
|
gt_points = [(p["x"], p["y"]) for p in s["points"]]
|
|
bg = Path(f"tcaptcha/data/raw/{sid}_bg.png")
|
|
prompt = Path(f"tcaptcha/data/raw/{sid}_prompt.png")
|
|
|
|
if not bg.exists() or not prompt.exists():
|
|
logger.warning(f"[{idx}/{total_samples}] {sid} 缺失图像文件")
|
|
continue
|
|
|
|
pred_points = solver.solve(bg, prompt)
|
|
if len(pred_points) < 3:
|
|
logger.warning(f"[{idx}/{total_samples}] {sid} 预测点数不足3个: {pred_points}")
|
|
failed_samples.append((sid, "insufficient_points", gt_points, pred_points))
|
|
continue
|
|
|
|
# 1. 严格顺序匹配评估
|
|
seq_hit = True
|
|
sample_point_hits = 0
|
|
distances = []
|
|
for i in range(3):
|
|
gx, gy = gt_points[i]
|
|
px, py = pred_points[i]
|
|
dist = np.hypot(px - gx, py - gy)
|
|
distances.append(dist)
|
|
if dist <= tolerance_px:
|
|
sample_point_hits += 1
|
|
point_errors.append(dist)
|
|
else:
|
|
seq_hit = False
|
|
|
|
total_points_hit += sample_point_hits
|
|
|
|
# 2. 无序匹配评估 (检查是否3个图标都找出来了，哪怕顺序搞错)
|
|
# 为每个预测点找最近的GT点
|
|
used_gt = set()
|
|
for px, py in pred_points:
|
|
best_gt = None
|
|
min_d = 9999
|
|
for gi, (gx, gy) in enumerate(gt_points):
|
|
if gi in used_gt:
|
|
continue
|
|
d = np.hypot(px - gx, py - gy)
|
|
if d < min_d:
|
|
min_d = d
|
|
best_gt = gi
|
|
if best_gt is not None and min_d <= tolerance_px:
|
|
used_gt.add(best_gt)
|
|
|
|
all_detected = (len(used_gt) == 3)
|
|
if all_detected:
|
|
any_order_success += 1
|
|
|
|
if seq_hit:
|
|
exact_seq_success += 1
|
|
status = "✅ PASS"
|
|
else:
|
|
status = "❌ FAIL"
|
|
failed_samples.append((sid, f"dist={[round(d,1) for d in distances]}", gt_points, pred_points))
|
|
|
|
print(f"[{idx:02d}/{total_samples}] {sid}: {status} | 点命中: {sample_point_hits}/3 | 距离: {[round(d, 1) for d in distances]}")
|
|
|
|
# 统计总结
|
|
seq_acc = exact_seq_success / total_samples * 100
|
|
order_agnostic_acc = any_order_success / total_samples * 100
|
|
pt_hit_rate = total_points_hit / total_points * 100
|
|
avg_err = np.mean(point_errors) if point_errors else 0.0
|
|
|
|
print("\n" + "=" * 60)
|
|
print("🎯 评估结果报告 (Evaluation Report):")
|
|
print(f" - 样本总数 (Total Samples): {total_samples}")
|
|
print(f" - 判定容差半径 (Tolerance): <= {tolerance_px} px")
|
|
print(f" - 单点命中率 (Point Hit Rate): {total_points_hit}/{total_points} ({pt_hit_rate:.2f}%)")
|
|
print(f" - 无序召回率 (Order-agnostic 3-Point Hit Rate): {any_order_success}/{total_samples} ({order_agnostic_acc:.2f}%)")
|
|
print(f" - 🌟 严格时序三点识别率 (Sequence Accuracy): {exact_seq_success}/{total_samples} ({seq_acc:.2f}%)")
|
|
print(f" - 命中点平均中心偏差 (Avg Center Offset): {avg_err:.2f} px")
|
|
print("=" * 60)
|
|
|
|
if failed_samples:
|
|
print(f"\n❌ 失败样本分析 (共 {len(failed_samples)} 个):")
|
|
for sid, reason, gt, pred in failed_samples[:10]:
|
|
print(f" - {sid}: {reason} | GT={gt} vs Pred={pred}")
|
|
if len(failed_samples) > 10:
|
|
print(f" ... 还有 {len(failed_samples) - 10} 个失败样本未列出")
|
|
|
|
return {
|
|
"seq_acc": seq_acc,
|
|
"order_agnostic_acc": order_agnostic_acc,
|
|
"pt_hit_rate": pt_hit_rate,
|
|
"failed_count": len(failed_samples)
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
tol = float(sys.argv[1]) if len(sys.argv) > 1 else 35.0
|
|
evaluate_solver(tolerance_px=tol)
|