hello_js_reverse_skill/tcaptcha/icon_click_solver.py

# -*- coding: utf-8 -*-
"""
腾讯天御『图标/图文混合点选』求解器 (subcapclass 2408 / 2401 之外的图形点选)

适用题型 (1ms.run aid=195672344 实测，2026-10-11):
  提示条 `img_index=0` 是 170x50 的**目标原图条**(按点击顺序排列, 例如: "0" + 图标 + "7"),
  背景图 `img_index=1` 是 672x480 的实景照片, 上面画着若干**纯黑轮廓**的符号
  (目标 + 干扰项), 部分目标被**旋转/倾斜**。

与 `glyph_solver.TextClickSolver`(汉字点选 2404) 的区别:
  * 2404: 指令是纯文字 ("请依次点击：本 车 憋"), 背景是**彩色**(橙黄)汉字, 用 HSV 分割 + OCR
  * 本类: 指令是**图标/数字**, 提示以**图片**给出, 背景符号是**纯黑轮廓**, 且会被旋转

算法 (不重训分类器, 直接利用"提示条给的就是目标原图"这一事实):
  1. 背景/提示条都按"近黑像素"分割成字形掩膜 (符号是纯黑, 背景是照片)
  2. 提示条连通域 = 有序目标模板 (按 x 排序)
  3. 背景连通域 = 候选符号
  4. 每个目标 vs 每个候选: 二值掩膜归一化(等比缩放到正方形)后做
     **旋转搜索 + IoU** 相似度 (旋转不变, 正好对付倾斜干扰)
  5. 用匈牙利/贪心做**一对一**指派, 按提示顺序输出点击坐标

坐标口径: 输出与腾讯 `ans` 相同的**原图坐标系 (672x480)**。
"""
from __future__ import annotations

import io
from dataclasses import dataclass
from typing import List, Optional, Sequence, Tuple

import cv2
import numpy as np
from PIL import Image

# 近黑阈值: 符号是纯黑(<=60), 照片最暗处也远高于此; 用"三通道都暗"避免蓝色天空误检
DARK_MAX = 95
MERGE_KERNEL = 5
MIN_AREA = 80
MAX_AREA = 9000


def _to_bgr(image) -> np.ndarray:
    if isinstance(image, np.ndarray):
        return image
    if isinstance(image, (bytes, bytearray)):
        arr = np.frombuffer(image, np.uint8)
        return cv2.imdecode(arr, cv2.IMREAD_COLOR)
    return cv2.cvtColor(np.asarray(image.convert("RGB")), cv2.COLOR_RGB2BGR)


def glyph_mask(bgr: np.ndarray, dark_max: int = DARK_MAX, kernel: int = MERGE_KERNEL,
               dilate: int = 3) -> np.ndarray:
    """近黑像素 -> 掩膜 (取三通道最大值, 只有真正的黑色轮廓会通过)。

    kernel>1 时做闭运算合并同一符号的笔画; dilate>0 时再膨胀一点 (分割阶段用)。
    """
    mx = bgr.max(axis=2)
    m = (mx < dark_max).astype(np.uint8) * 255
    if kernel > 1:
        m = cv2.morphologyEx(m, cv2.MORPH_CLOSE,
                             cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel, kernel)))
    if dilate > 0:
        m = cv2.dilate(m, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (dilate, dilate)))
    return m


def components(mask: np.ndarray, min_area: int = MIN_AREA, max_area: int = MAX_AREA) -> List[Tuple[int, int, int, int, int]]:
    n, lab, stats, _ = cv2.connectedComponentsWithStats(mask)
    out = []
    for i in range(1, n):
        x, y, w, h, a = (int(v) for v in stats[i])
        if a < min_area or a > max_area:
            continue
        if w < 6 or h < 6:
            continue
        out.append((x, y, w, h, a))
    return out


def _norm(bgr: np.ndarray, box, size: int = 64) -> np.ndarray:
    """把 box 内的符号归一化成 size x size 的二值掩膜 (等比居中, 不拉伸)。

    注意: 画布必须用**白色**填充 —— 黑色会被"近黑"掩膜当成字形 (踩过这个坑)。
    """
    x, y, w, h, _ = box
    crop = bgr[y:y + h, x:x + w]
    s = max(w, h)
    canvas = np.full((s, s, 3), 255, np.uint8)
    canvas[(s - h) // 2:(s - h) // 2 + h, (s - w) // 2:(s - w) // 2 + w] = crop
    small = cv2.resize(canvas, (size, size), interpolation=cv2.INTER_AREA)
    return glyph_mask(small, dark_max=140, kernel=3, dilate=0) > 127


def _rotated(mask: np.ndarray, ang: float, size: int = 64) -> np.ndarray:
    M = cv2.getRotationMatrix2D((size / 2, size / 2), ang, 1.0)
    r = cv2.warpAffine(mask.astype(np.uint8) * 255, M, (size, size))
    return r > 127


def _iou(a: np.ndarray, b: np.ndarray) -> float:
    inter = np.logical_and(a, b).sum()
    union = np.logical_or(a, b).sum()
    return float(inter) / float(union) if union else 0.0


@dataclass
class IconClickResult:
    points: List[Tuple[int, int]]           # 原图坐标 (672x480 口径)
    scores: List[float]
    angles: List[float]
    boxes: List[Tuple[int, int, int, int]]
    target_count: int

    @property
    def ok(self) -> bool:
        return len(self.points) == self.target_count and self.target_count > 0

    @property
    def min_score(self) -> float:
        return min(self.scores) if self.scores else 0.0

    def describe(self) -> str:
        return " ".join("(%d,%d|%.2f@%d°)" % (p[0], p[1], s, a)
                        for p, s, a in zip(self.points, self.scores, self.angles))


class IconClickSolver:
    """提示图(目标条) + 背景图 -> 有序点击坐标。"""

    def __init__(self, angles: Sequence[float] = tuple(range(-45, 46, 5)),
                 size: int = 64, min_area: int = MIN_AREA):
        self.angles = tuple(angles)
        self.size = size
        self.min_area = min_area

    def solve(self, bg, prompt) -> IconClickResult:
        bg_bgr = _to_bgr(bg)
        pr_bgr = _to_bgr(prompt)

        targets = sorted(components(glyph_mask(pr_bgr), self.min_area), key=lambda b: b[0])
        cands = components(glyph_mask(bg_bgr), self.min_area)
        if not targets or not cands:
            return IconClickResult([], [], [], [], len(targets))

        tm = [_norm(pr_bgr, b, self.size) for b in targets]
        cm = [_norm(bg_bgr, b, self.size) for b in cands]
        # 候选旋转缓存
        rot = [[(a, _rotated(c, a, self.size)) for a in self.angles] for c in cm]

        # 得分矩阵: 每个目标在 (候选, 角度) 上的最大 IoU
        S = np.zeros((len(tm), len(cm)), np.float32)
        A = np.zeros((len(tm), len(cm)), np.float32)
        for ti, t in enumerate(tm):
            for ci, rs in enumerate(rot):
                best = (-1.0, 0.0)
                for ang, r in rs:
                    v = _iou(t, r)
                    if v > best[0]:
                        best = (v, ang)
                S[ti, ci], A[ti, ci] = best

        # 贪心一对一指派 (按全局得分从高到低)
        pairs = sorted(((S[i, j], i, j) for i in range(len(tm)) for j in range(len(cm))),
                       key=lambda x: -x[0])
        used_t, used_c = set(), set()
        assign = {}
        for v, i, j in pairs:
            if i in used_t or j in used_c:
                continue
            used_t.add(i)
            used_c.add(j)
            assign[i] = (j, v, A[i, j])

        points, scores, angs, boxes = [], [], [], []
        for i in range(len(tm)):
            if i not in assign:
                continue
            j, v, ang = assign[i]
            x, y, w, h, _ = cands[j]
            points.append((x + w // 2, y + h // 2))
            scores.append(v)
            angs.append(ang)
            boxes.append((x, y, w, h))
        return IconClickResult(points, scores, angs, boxes, len(tm))

    def annotate(self, bg, result: IconClickResult) -> np.ndarray:
        """把结果画到背景图上 (自检用)。"""
        img = _to_bgr(bg).copy()
        for idx, ((px, py), sc) in enumerate(zip(result.points, result.scores), 1):
            cv2.circle(img, (px, py), 26, (0, 0, 255), 3)
            cv2.putText(img, str(idx), (px - 8, py + 8), cv2.FONT_HERSHEY_SIMPLEX,
                        1.0, (0, 255, 0), 2)
        return img


if __name__ == "__main__":
    import sys
    bg = cv2.imread(sys.argv[1] if len(sys.argv) > 1 else "/tmp/1mscap/bg.jpg")
    pr = cv2.imread(sys.argv[2] if len(sys.argv) > 2 else "/tmp/1mscap/prompt.jpg")
    s = IconClickSolver()
    r = s.solve(bg, pr)
    print("ok=%s points=%s scores=%s angles=%s" % (r.ok, r.points, [round(x, 3) for x in r.scores], r.angles))
    out = s.annotate(bg, r)
    cv2.imwrite("/tmp/1mscap/annotated.jpg", out)
    print("annotated -> /tmp/1mscap/annotated.jpg")