282 lines
9.9 KiB
Python
282 lines
9.9 KiB
Python
"""虎牙滑块/点选验证码 OCR。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from functools import lru_cache
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
from typing import cast
|
|
|
|
import cv2
|
|
import numpy as np
|
|
import onnxruntime as ort
|
|
from PIL import Image
|
|
from scipy.optimize import linear_sum_assignment
|
|
|
|
MODEL_DIR = Path(__file__).resolve().parent / "models"
|
|
|
|
|
|
class YoloOnnx:
|
|
"""轻量 YOLO ONNX 推理封装。"""
|
|
|
|
def __init__(
|
|
self,
|
|
model_path: str | Path,
|
|
classes: list[str],
|
|
providers: list[str] | None = None,
|
|
):
|
|
providers = providers or ["CPUExecutionProvider"]
|
|
ort.set_default_logger_severity(3)
|
|
self.session = ort.InferenceSession(str(model_path), providers=providers)
|
|
self.input_name = self.session.get_inputs()[0].name
|
|
self.output_name = self.session.get_outputs()[0].name
|
|
self.input_shape = self.session.get_inputs()[0].shape[2:]
|
|
self.names = classes
|
|
self.scale = 1.0
|
|
self.pad_top = 0
|
|
self.pad_left = 0
|
|
|
|
@staticmethod
|
|
def xywh2xyxy(value: np.ndarray) -> np.ndarray:
|
|
result = np.copy(value)
|
|
result[..., 0] = value[..., 0] - value[..., 2] / 2
|
|
result[..., 1] = value[..., 1] - value[..., 3] / 2
|
|
result[..., 2] = value[..., 0] + value[..., 2] / 2
|
|
result[..., 3] = value[..., 1] + value[..., 3] / 2
|
|
return result
|
|
|
|
def preprocess(self, image: Image.Image) -> np.ndarray:
|
|
img = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
|
|
height, width = img.shape[:2]
|
|
|
|
self.scale = min(self.input_shape[0] / height, self.input_shape[1] / width)
|
|
new_size = (int(width * self.scale), int(height * self.scale))
|
|
resized = cv2.resize(img, new_size, interpolation=cv2.INTER_LINEAR)
|
|
|
|
dh = self.input_shape[0] - new_size[1]
|
|
dw = self.input_shape[1] - new_size[0]
|
|
self.pad_top = dh // 2
|
|
pad_bottom = dh - self.pad_top
|
|
self.pad_left = dw // 2
|
|
pad_right = dw - self.pad_left
|
|
|
|
padded = cv2.copyMakeBorder(
|
|
resized,
|
|
self.pad_top,
|
|
pad_bottom,
|
|
self.pad_left,
|
|
pad_right,
|
|
cv2.BORDER_CONSTANT,
|
|
value=(114, 114, 114),
|
|
)
|
|
input_tensor = padded.transpose(2, 0, 1).astype(np.float32) / 255.0
|
|
return np.expand_dims(input_tensor, axis=0)
|
|
|
|
@staticmethod
|
|
def nms(
|
|
boxes: np.ndarray, scores: np.ndarray, iou_threshold: float = 0.3
|
|
) -> list[int]:
|
|
order = scores.argsort()[::-1]
|
|
keep = []
|
|
|
|
while order.size > 0:
|
|
index = order[0]
|
|
keep.append(index)
|
|
|
|
xx1 = np.maximum(boxes[index, 0], boxes[order[1:], 0])
|
|
yy1 = np.maximum(boxes[index, 1], boxes[order[1:], 1])
|
|
xx2 = np.minimum(boxes[index, 2], boxes[order[1:], 2])
|
|
yy2 = np.minimum(boxes[index, 3], boxes[order[1:], 3])
|
|
|
|
width = np.maximum(0.0, xx2 - xx1)
|
|
height = np.maximum(0.0, yy2 - yy1)
|
|
intersection = width * height
|
|
area_i = (boxes[index, 2] - boxes[index, 0]) * (
|
|
boxes[index, 3] - boxes[index, 1]
|
|
)
|
|
area_j = (boxes[order[1:], 2] - boxes[order[1:], 0]) * (
|
|
boxes[order[1:], 3] - boxes[order[1:], 1]
|
|
)
|
|
iou = intersection / (area_i + area_j - intersection + 1e-7)
|
|
|
|
inds = np.where(iou <= iou_threshold)[0]
|
|
order = order[inds + 1]
|
|
|
|
return keep
|
|
|
|
def postprocess(
|
|
self,
|
|
outputs: list[np.ndarray],
|
|
conf_threshold: float = 0.1,
|
|
iou_threshold: float = 0.3,
|
|
) -> list[dict]:
|
|
predictions = np.squeeze(outputs[0]).T
|
|
class_scores = predictions[:, 4:]
|
|
scores = np.max(class_scores, axis=1)
|
|
class_ids = np.argmax(class_scores, axis=1)
|
|
|
|
valid_indices = scores > conf_threshold
|
|
predictions = predictions[valid_indices]
|
|
scores = scores[valid_indices]
|
|
class_ids = class_ids[valid_indices]
|
|
if predictions.size == 0:
|
|
return []
|
|
|
|
boxes = predictions[:, :4].copy()
|
|
boxes[:, 0] = (boxes[:, 0] - self.pad_left) / self.scale
|
|
boxes[:, 1] = (boxes[:, 1] - self.pad_top) / self.scale
|
|
boxes[:, 2] /= self.scale
|
|
boxes[:, 3] /= self.scale
|
|
xyxy_boxes = self.xywh2xyxy(boxes)
|
|
|
|
keep_indices = []
|
|
for class_id in np.unique(class_ids):
|
|
class_indices = np.where(class_ids == class_id)[0]
|
|
class_keep = self.nms(
|
|
xyxy_boxes[class_indices],
|
|
scores[class_indices],
|
|
iou_threshold,
|
|
)
|
|
keep_indices.extend(class_indices[class_keep])
|
|
|
|
results = []
|
|
for index in keep_indices:
|
|
class_id = int(class_ids[index])
|
|
x1, y1, x2, y2 = map(int, xyxy_boxes[index].tolist())
|
|
results.append(
|
|
{
|
|
"label_id": class_id,
|
|
"label_name": self.names[class_id],
|
|
"confidence": float(scores[index]),
|
|
"box_mid_xy": [(x1 + x2) // 2, (y1 + y2) // 2],
|
|
"xyxy": [x1, y1, x2, y2],
|
|
}
|
|
)
|
|
|
|
results.sort(key=lambda item: item["confidence"], reverse=True)
|
|
return results
|
|
|
|
def detect(self, image: Image.Image) -> list[dict]:
|
|
input_tensor = self.preprocess(image)
|
|
outputs = cast(
|
|
list[np.ndarray],
|
|
self.session.run([self.output_name], {self.input_name: input_tensor}),
|
|
)
|
|
return self.postprocess(outputs)
|
|
|
|
|
|
class SimilarityOnnx:
|
|
"""点选验证码中目标图和候选字的相似度模型。"""
|
|
|
|
def __init__(self, model_path: str | Path, providers: list[str] | None = None):
|
|
providers = providers or ["CPUExecutionProvider"]
|
|
ort.set_default_logger_severity(3)
|
|
self.session = ort.InferenceSession(str(model_path), providers=providers)
|
|
self.input_shape: tuple[int, int] = (64, 64)
|
|
|
|
@staticmethod
|
|
def sigmoid(value: np.ndarray) -> np.ndarray:
|
|
return 1 / (1 + np.exp(-value))
|
|
|
|
@staticmethod
|
|
def _to_image(value) -> Image.Image:
|
|
if isinstance(value, np.ndarray):
|
|
return Image.fromarray(value)
|
|
if isinstance(value, bytes):
|
|
return Image.open(BytesIO(value))
|
|
if isinstance(value, Image.Image):
|
|
return value
|
|
return Image.open(value)
|
|
|
|
def _tensor(self, value) -> np.ndarray:
|
|
image = (
|
|
self._to_image(value)
|
|
.convert("RGB")
|
|
.resize((self.input_shape[1], self.input_shape[0]), 1)
|
|
)
|
|
array = np.array(image).astype(np.float32) / 255.0
|
|
return np.expand_dims(np.transpose(array, (2, 0, 1)), 0)
|
|
|
|
def score(self, image_1, image_2) -> int:
|
|
out = self.session.run(
|
|
None, {"x1": self._tensor(image_1), "x2": self._tensor(image_2)}
|
|
)
|
|
similarity = self.sigmoid(cast(np.ndarray, out[0]))[0][0]
|
|
return int(round(similarity.item(), 2) * 100)
|
|
|
|
|
|
class HuyaCaptchaOcr:
|
|
"""封装虎牙滑块和点选识别。"""
|
|
|
|
def __init__(
|
|
self, model_dir: str | Path = MODEL_DIR, providers: list[str] | None = None
|
|
):
|
|
model_dir = Path(model_dir)
|
|
self.similarity = SimilarityOnnx(
|
|
model_dir / "weights.onnx", providers=providers
|
|
)
|
|
self.click_model = YoloOnnx(
|
|
model_dir / "best.onnx", classes=["target", "char"], providers=providers
|
|
)
|
|
self.slider_model = YoloOnnx(
|
|
model_dir / "slider_2.onnx", classes=["slider"], providers=providers
|
|
)
|
|
|
|
@staticmethod
|
|
def _open_image(image_bytes: bytes) -> Image.Image:
|
|
return Image.open(BytesIO(image_bytes)).convert("RGB")
|
|
|
|
def ocr_slider_box(self, image_bytes: bytes) -> list[int]:
|
|
"""识别滑块缺口框，返回 [x1, y1, x2, y2]。"""
|
|
image = self._open_image(image_bytes)
|
|
detections = self.slider_model.detect(image)
|
|
boxes = [item["xyxy"] for item in detections]
|
|
return boxes[0] if boxes else [100, 100, 100, 100]
|
|
|
|
def ocr_click_points(self, image_bytes: bytes) -> list[list[int]]:
|
|
"""识别点选坐标，返回按目标顺序排列的点击点。"""
|
|
image = self._open_image(image_bytes)
|
|
detections = self.click_model.detect(image)
|
|
results = [
|
|
{**item, "cropped_image": image.crop(tuple(item["xyxy"]))}
|
|
for item in detections
|
|
]
|
|
|
|
char_list = [item for item in results if item.get("label_name") == "char"]
|
|
target_list = [item for item in results if item.get("label_name") == "target"]
|
|
target_list = [
|
|
item for item in target_list if item["xyxy"][2] - item["xyxy"][0] > 10
|
|
]
|
|
char_list.sort(key=lambda item: item["xyxy"][0])
|
|
target_list.sort(key=lambda item: item["xyxy"][0])
|
|
|
|
if not char_list or not target_list:
|
|
return []
|
|
|
|
score_matrix = np.zeros((len(target_list), len(char_list)))
|
|
for target_index, target in enumerate(target_list):
|
|
for char_index, char in enumerate(char_list):
|
|
try:
|
|
score_matrix[target_index][char_index] = -self.similarity.score(
|
|
target["cropped_image"],
|
|
char["cropped_image"],
|
|
)
|
|
except Exception:
|
|
score_matrix[target_index][char_index] = 1e6
|
|
|
|
row_ind, col_ind = linear_sum_assignment(score_matrix)
|
|
matched = []
|
|
for target_index, char_index in zip(row_ind, col_ind):
|
|
if score_matrix[target_index][char_index] == 1e6:
|
|
continue
|
|
char = char_list[char_index]
|
|
matched.append({**char, "index": target_index + 1})
|
|
|
|
return [item["box_mid_xy"] for item in matched]
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def default_ocr() -> HuyaCaptchaOcr:
|
|
"""懒加载默认 OCR，避免导入 core.huya 时立即加载 150MB 模型。"""
|
|
return HuyaCaptchaOcr()
|