perf(douyu): 消除任务锁竞争并增加创建背压
This commit is contained in:
@@ -6,7 +6,7 @@ import asyncio
|
||||
import threading
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -59,6 +59,38 @@ DOUYU_PROXY_TASK_TYPES = {
|
||||
}
|
||||
|
||||
|
||||
_DOUYU_RESULT_DEBUG_KEYS = {
|
||||
"raw",
|
||||
"query_params",
|
||||
"qq_app_scheme",
|
||||
"role_param_url",
|
||||
"game_role_bind_url",
|
||||
}
|
||||
|
||||
|
||||
def slim_douyu_task_result_for_storage(result: dict | None) -> dict | None:
|
||||
"""去掉任务结果里的调试大字段,避免每次轮询重复写大 JSON。
|
||||
|
||||
运行器在内存中仍使用完整 result;这里只影响落库和 WS 展示。二维码 URL、
|
||||
角色状态、积分、订单号等业务字段全部保留。
|
||||
"""
|
||||
if result is None:
|
||||
return None
|
||||
|
||||
def clean(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: clean(item)
|
||||
for key, item in value.items()
|
||||
if key not in _DOUYU_RESULT_DEBUG_KEYS
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [clean(item) for item in value]
|
||||
return value
|
||||
|
||||
return clean(result)
|
||||
|
||||
|
||||
class DouyuBatchRunnerCore:
|
||||
"""斗鱼任务执行器公共基础:批次状态、日志、任务落库与客户端构造。"""
|
||||
|
||||
@@ -196,9 +228,7 @@ class DouyuBatchRunnerCore:
|
||||
|
||||
sync_task_from_binding_runner(db, douyu_task_id=int(task.id))
|
||||
except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果
|
||||
logger.exception(
|
||||
"[douyu] 同步直播账号履约状态失败: task_id={}", task.id
|
||||
)
|
||||
logger.exception("[douyu] 同步直播账号履约状态失败: task_id={}", task.id)
|
||||
|
||||
def _sync_platform_fulfillment_operation(
|
||||
self,
|
||||
@@ -220,9 +250,7 @@ class DouyuBatchRunnerCore:
|
||||
message=message,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - 履约时间线不能覆盖任务执行结果
|
||||
logger.exception(
|
||||
"[douyu] 写入履约操作时间线失败: task_id={}", task.id
|
||||
)
|
||||
logger.exception("[douyu] 写入履约操作时间线失败: task_id={}", task.id)
|
||||
|
||||
def _mark_task(
|
||||
self,
|
||||
@@ -235,30 +263,28 @@ class DouyuBatchRunnerCore:
|
||||
task.status = status
|
||||
task.message = message[:512]
|
||||
if result is not None:
|
||||
task.result = result
|
||||
task.result = slim_douyu_task_result_for_storage(result)
|
||||
task.finished_at = datetime.now(UTC)
|
||||
if task.account_id and status in {"success", "failed", "error", "stopped"}:
|
||||
from .business_events import record_douyu_task_result
|
||||
|
||||
account = (
|
||||
db.query(Account).filter(Account.id == task.account_id).first()
|
||||
)
|
||||
if account is not None:
|
||||
record_douyu_task_result(
|
||||
db,
|
||||
account,
|
||||
task_id=int(task.id),
|
||||
task_type=str(task.task_type or ""),
|
||||
task_label=SUPPORTED_DOUYU_TASK_TYPES.get(
|
||||
str(task.task_type or ""),
|
||||
str(task.task_type or "斗鱼任务"),
|
||||
),
|
||||
scope=str(task.handbook_scope or ""),
|
||||
status=status,
|
||||
message=message,
|
||||
occurred_at=task.finished_at,
|
||||
actor_id=int(task.created_by),
|
||||
)
|
||||
account = (
|
||||
db.query(Account).filter(Account.id == task.account_id).first()
|
||||
if task.account_id and status in {"success", "failed", "error", "stopped"}
|
||||
else None
|
||||
)
|
||||
event_context = (
|
||||
{
|
||||
"task_id": int(task.id),
|
||||
"task_type": str(task.task_type or ""),
|
||||
"task_label": SUPPORTED_DOUYU_TASK_TYPES.get(
|
||||
str(task.task_type or ""),
|
||||
str(task.task_type or "斗鱼任务"),
|
||||
),
|
||||
"scope": str(task.handbook_scope or ""),
|
||||
"finished_at": task.finished_at,
|
||||
"created_by": int(task.created_by),
|
||||
}
|
||||
if account is not None
|
||||
else None
|
||||
)
|
||||
self._sync_platform_fulfillment_operation(
|
||||
db,
|
||||
task,
|
||||
@@ -267,6 +293,28 @@ class DouyuBatchRunnerCore:
|
||||
)
|
||||
db.commit()
|
||||
self._push_task_event(task)
|
||||
if account is not None and event_context is not None:
|
||||
try:
|
||||
from .business_events import record_douyu_task_result
|
||||
|
||||
record_douyu_task_result(
|
||||
db,
|
||||
account,
|
||||
task_id=event_context["task_id"],
|
||||
task_type=event_context["task_type"],
|
||||
task_label=event_context["task_label"],
|
||||
scope=event_context["scope"],
|
||||
status=status,
|
||||
message=message,
|
||||
occurred_at=event_context["finished_at"],
|
||||
actor_id=event_context["created_by"],
|
||||
)
|
||||
db.commit()
|
||||
except Exception: # noqa: BLE001 - 时间线旁路不能覆盖已提交的任务结果
|
||||
db.rollback()
|
||||
logger.exception(
|
||||
"[douyu] 写入账号任务结果事件失败: task_id={}", task.id
|
||||
)
|
||||
self._sync_platform_fulfillment(db, task)
|
||||
|
||||
def _update_task_progress(
|
||||
@@ -280,7 +328,7 @@ class DouyuBatchRunnerCore:
|
||||
task.status = status
|
||||
task.message = message[:512]
|
||||
if result is not None:
|
||||
task.result = result
|
||||
task.result = slim_douyu_task_result_for_storage(result)
|
||||
db.commit()
|
||||
self._push_task_event(task)
|
||||
self._sync_platform_fulfillment(db, task)
|
||||
|
||||
Reference in New Issue
Block a user