# -*- coding: utf-8 -*- """ 腾讯天御『图标/图文混合点选』求解器 (subcapclass 2408 / 2401 之外的图形点选) 适用题型 (1ms.run aid=195672344 实测，2026-10-11): 提示条 `img_index=0` 是 170x50 的**目标原图条**(按点击顺序排列, 例如: "0" + 图标 + "7"), 背景图 `img_index=1` 是 672x480 的实景照片, 上面画着若干**纯黑轮廓**的符号 (目标 + 干扰项), 部分目标被**旋转/倾斜**。 与 `glyph_solver.TextClickSolver`(汉字点选 2404) 的区别: * 2404: 指令是纯文字 ("请依次点击：本 车 憋"), 背景是**彩色**(橙黄)汉字, 用 HSV 分割 + OCR * 本类: 指令是**图标/数字**, 提示以**图片**给出, 背景符号是**纯黑轮廓**, 且会被旋转 算法 (不重训分类器, 直接利用"提示条给的就是目标原图"这一事实): 1. 背景/提示条都按"近黑像素"分割成字形掩膜 (符号是纯黑, 背景是照片) 2. 提示条连通域 = 有序目标模板 (按 x 排序) 3. 背景连通域 = 候选符号 4. 每个目标 vs 每个候选: 二值掩膜归一化(等比缩放到正方形)后做 **旋转搜索 + IoU** 相似度 (旋转不变, 正好对付倾斜干扰) 5. 用匈牙利/贪心做**一对一**指派, 按提示顺序输出点击坐标 坐标口径: 输出与腾讯 `ans` 相同的**原图坐标系 (672x480)**。 """ from __future__ import annotations import io from dataclasses import dataclass from typing import List, Optional, Sequence, Tuple import cv2 import numpy as np from PIL import Image # 近黑阈值: 符号是纯黑(<=60), 照片最暗处也远高于此; 用"三通道都暗"避免蓝色天空误检 DARK_MAX = 95 MERGE_KERNEL = 5 MIN_AREA = 80 MAX_AREA = 9000 def _to_bgr(image) -> np.ndarray: if isinstance(image, np.ndarray): return image if isinstance(image, (bytes, bytearray)): arr = np.frombuffer(image, np.uint8) return cv2.imdecode(arr, cv2.IMREAD_COLOR) return cv2.cvtColor(np.asarray(image.convert("RGB")), cv2.COLOR_RGB2BGR) def glyph_mask(bgr: np.ndarray, dark_max: int = DARK_MAX, kernel: int = MERGE_KERNEL, dilate: int = 3) -> np.ndarray: """近黑像素 -> 掩膜 (取三通道最大值, 只有真正的黑色轮廓会通过)。 kernel>1 时做闭运算合并同一符号的笔画; dilate>0 时再膨胀一点 (分割阶段用)。 """ mx = bgr.max(axis=2) m = (mx < dark_max).astype(np.uint8) * 255 if kernel > 1: m = cv2.morphologyEx(m, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel, kernel))) if dilate > 0: m = cv2.dilate(m, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (dilate, dilate))) return m def components(mask: np.ndarray, min_area: int = MIN_AREA, max_area: int = MAX_AREA) -> List[Tuple[int, int, int, int, int]]: n, lab, stats, _ = cv2.connectedComponentsWithStats(mask) out = [] for i in range(1, n): x, y, w, h, a = (int(v) for v in stats[i]) if a < min_area or a > max_area: continue if w < 6 or h < 6: continue out.append((x, y, w, h, a)) return out def _norm(bgr: np.ndarray, box, size: int = 64) -> np.ndarray: """把 box 内的符号归一化成 size x size 的二值掩膜 (等比居中, 不拉伸)。 注意: 画布必须用**白色**填充 —— 黑色会被"近黑"掩膜当成字形 (踩过这个坑)。 """ x, y, w, h, _ = box crop = bgr[y:y + h, x:x + w] s = max(w, h) canvas = np.full((s, s, 3), 255, np.uint8) canvas[(s - h) // 2:(s - h) // 2 + h, (s - w) // 2:(s - w) // 2 + w] = crop small = cv2.resize(canvas, (size, size), interpolation=cv2.INTER_AREA) return glyph_mask(small, dark_max=140, kernel=3, dilate=0) > 127 def _rotated(mask: np.ndarray, ang: float, size: int = 64) -> np.ndarray: M = cv2.getRotationMatrix2D((size / 2, size / 2), ang, 1.0) r = cv2.warpAffine(mask.astype(np.uint8) * 255, M, (size, size)) return r > 127 def _iou(a: np.ndarray, b: np.ndarray) -> float: inter = np.logical_and(a, b).sum() union = np.logical_or(a, b).sum() return float(inter) / float(union) if union else 0.0 @dataclass class IconClickResult: points: List[Tuple[int, int]] # 原图坐标 (672x480 口径) scores: List[float] angles: List[float] boxes: List[Tuple[int, int, int, int]] target_count: int @property def ok(self) -> bool: return len(self.points) == self.target_count and self.target_count > 0 @property def min_score(self) -> float: return min(self.scores) if self.scores else 0.0 def describe(self) -> str: return " ".join("(%d,%d|%.2f@%d°)" % (p[0], p[1], s, a) for p, s, a in zip(self.points, self.scores, self.angles)) class IconClickSolver: """提示图(目标条) + 背景图 -> 有序点击坐标。""" def __init__(self, angles: Sequence[float] = tuple(range(-45, 46, 5)), size: int = 64, min_area: int = MIN_AREA): self.angles = tuple(angles) self.size = size self.min_area = min_area def solve(self, bg, prompt) -> IconClickResult: bg_bgr = _to_bgr(bg) pr_bgr = _to_bgr(prompt) targets = sorted(components(glyph_mask(pr_bgr), self.min_area), key=lambda b: b[0]) cands = components(glyph_mask(bg_bgr), self.min_area) if not targets or not cands: return IconClickResult([], [], [], [], len(targets)) tm = [_norm(pr_bgr, b, self.size) for b in targets] cm = [_norm(bg_bgr, b, self.size) for b in cands] # 候选旋转缓存 rot = [[(a, _rotated(c, a, self.size)) for a in self.angles] for c in cm] # 得分矩阵: 每个目标在 (候选, 角度) 上的最大 IoU S = np.zeros((len(tm), len(cm)), np.float32) A = np.zeros((len(tm), len(cm)), np.float32) for ti, t in enumerate(tm): for ci, rs in enumerate(rot): best = (-1.0, 0.0) for ang, r in rs: v = _iou(t, r) if v > best[0]: best = (v, ang) S[ti, ci], A[ti, ci] = best # 贪心一对一指派 (按全局得分从高到低) pairs = sorted(((S[i, j], i, j) for i in range(len(tm)) for j in range(len(cm))), key=lambda x: -x[0]) used_t, used_c = set(), set() assign = {} for v, i, j in pairs: if i in used_t or j in used_c: continue used_t.add(i) used_c.add(j) assign[i] = (j, v, A[i, j]) points, scores, angs, boxes = [], [], [], [] for i in range(len(tm)): if i not in assign: continue j, v, ang = assign[i] x, y, w, h, _ = cands[j] points.append((x + w // 2, y + h // 2)) scores.append(v) angs.append(ang) boxes.append((x, y, w, h)) return IconClickResult(points, scores, angs, boxes, len(tm)) def annotate(self, bg, result: IconClickResult) -> np.ndarray: """把结果画到背景图上 (自检用)。""" img = _to_bgr(bg).copy() for idx, ((px, py), sc) in enumerate(zip(result.points, result.scores), 1): cv2.circle(img, (px, py), 26, (0, 0, 255), 3) cv2.putText(img, str(idx), (px - 8, py + 8), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2) return img if __name__ == "__main__": import sys bg = cv2.imread(sys.argv[1] if len(sys.argv) > 1 else "/tmp/1mscap/bg.jpg") pr = cv2.imread(sys.argv[2] if len(sys.argv) > 2 else "/tmp/1mscap/prompt.jpg") s = IconClickSolver() r = s.solve(bg, pr) print("ok=%s points=%s scores=%s angles=%s" % (r.ok, r.points, [round(x, 3) for x in r.scores], r.angles)) out = s.annotate(bg, r) cv2.imwrite("/tmp/1mscap/annotated.jpg", out) print("annotated -> /tmp/1mscap/annotated.jpg")