perf(douyu): 消除任务锁竞争并增加创建背压

This commit is contained in:
yml
2026-11-19 22:01:21 +08:00
parent 62cb0a41ec
commit 09cdf3f2eb
11 changed files with 555 additions and 95 deletions
+79 -31
View File
@@ -6,7 +6,7 @@ import asyncio
import threading
import time
from datetime import UTC, datetime
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
from loguru import logger
from sqlalchemy.orm import Session
@@ -59,6 +59,38 @@ DOUYU_PROXY_TASK_TYPES = {
}
_DOUYU_RESULT_DEBUG_KEYS = {
"raw",
"query_params",
"qq_app_scheme",
"role_param_url",
"game_role_bind_url",
}
def slim_douyu_task_result_for_storage(result: dict | None) -> dict | None:
"""去掉任务结果里的调试大字段,避免每次轮询重复写大 JSON。
运行器在内存中仍使用完整 result;这里只影响落库和 WS 展示。二维码 URL、
角色状态、积分、订单号等业务字段全部保留。
"""
if result is None:
return None
def clean(value: Any) -> Any:
if isinstance(value, dict):
return {
key: clean(item)
for key, item in value.items()
if key not in _DOUYU_RESULT_DEBUG_KEYS
}
if isinstance(value, list):
return [clean(item) for item in value]
return value
return clean(result)
class DouyuBatchRunnerCore:
"""斗鱼任务执行器公共基础:批次状态、日志、任务落库与客户端构造。"""
@@ -196,9 +228,7 @@ class DouyuBatchRunnerCore:
sync_task_from_binding_runner(db, douyu_task_id=int(task.id))
except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果
logger.exception(
"[douyu] 同步直播账号履约状态失败: task_id={}", task.id
)
logger.exception("[douyu] 同步直播账号履约状态失败: task_id={}", task.id)
def _sync_platform_fulfillment_operation(
self,
@@ -220,9 +250,7 @@ class DouyuBatchRunnerCore:
message=message,
)
except Exception: # noqa: BLE001 - 履约时间线不能覆盖任务执行结果
logger.exception(
"[douyu] 写入履约操作时间线失败: task_id={}", task.id
)
logger.exception("[douyu] 写入履约操作时间线失败: task_id={}", task.id)
def _mark_task(
self,
@@ -235,30 +263,28 @@ class DouyuBatchRunnerCore:
task.status = status
task.message = message[:512]
if result is not None:
task.result = result
task.result = slim_douyu_task_result_for_storage(result)
task.finished_at = datetime.now(UTC)
if task.account_id and status in {"success", "failed", "error", "stopped"}:
from .business_events import record_douyu_task_result
account = (
db.query(Account).filter(Account.id == task.account_id).first()
)
if account is not None:
record_douyu_task_result(
db,
account,
task_id=int(task.id),
task_type=str(task.task_type or ""),
task_label=SUPPORTED_DOUYU_TASK_TYPES.get(
str(task.task_type or ""),
str(task.task_type or "斗鱼任务"),
),
scope=str(task.handbook_scope or ""),
status=status,
message=message,
occurred_at=task.finished_at,
actor_id=int(task.created_by),
)
account = (
db.query(Account).filter(Account.id == task.account_id).first()
if task.account_id and status in {"success", "failed", "error", "stopped"}
else None
)
event_context = (
{
"task_id": int(task.id),
"task_type": str(task.task_type or ""),
"task_label": SUPPORTED_DOUYU_TASK_TYPES.get(
str(task.task_type or ""),
str(task.task_type or "斗鱼任务"),
),
"scope": str(task.handbook_scope or ""),
"finished_at": task.finished_at,
"created_by": int(task.created_by),
}
if account is not None
else None
)
self._sync_platform_fulfillment_operation(
db,
task,
@@ -267,6 +293,28 @@ class DouyuBatchRunnerCore:
)
db.commit()
self._push_task_event(task)
if account is not None and event_context is not None:
try:
from .business_events import record_douyu_task_result
record_douyu_task_result(
db,
account,
task_id=event_context["task_id"],
task_type=event_context["task_type"],
task_label=event_context["task_label"],
scope=event_context["scope"],
status=status,
message=message,
occurred_at=event_context["finished_at"],
actor_id=event_context["created_by"],
)
db.commit()
except Exception: # noqa: BLE001 - 时间线旁路不能覆盖已提交的任务结果
db.rollback()
logger.exception(
"[douyu] 写入账号任务结果事件失败: task_id={}", task.id
)
self._sync_platform_fulfillment(db, task)
def _update_task_progress(
@@ -280,7 +328,7 @@ class DouyuBatchRunnerCore:
task.status = status
task.message = message[:512]
if result is not None:
task.result = result
task.result = slim_douyu_task_result_for_storage(result)
db.commit()
self._push_task_event(task)
self._sync_platform_fulfillment(db, task)