perf(douyu): 消除任务锁竞争并增加创建背压
This commit is contained in:
@@ -6,6 +6,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import threading
|
||||
|
||||
from web.backend.database import SessionLocal
|
||||
@@ -13,16 +14,33 @@ from web.backend.database import SessionLocal
|
||||
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
|
||||
|
||||
|
||||
def _env_positive_int(name: str, default: int) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
return value if value > 0 else default
|
||||
|
||||
|
||||
DOUYU_BATCH_CREATE_CONCURRENCY = _env_positive_int(
|
||||
"DOUYU_BATCH_CREATE_CONCURRENCY",
|
||||
2,
|
||||
)
|
||||
DOUYU_MAX_ACTIVE_BATCHES = _env_positive_int("DOUYU_MAX_ACTIVE_BATCHES", 20)
|
||||
|
||||
|
||||
def start_douyu_batch(
|
||||
*,
|
||||
batch_id: str,
|
||||
task_type: str,
|
||||
payload: dict,
|
||||
concurrency: int,
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
log_queue: asyncio.Queue | None = None,
|
||||
) -> None:
|
||||
"""必须在运行中的 asyncio 事件循环里调用。"""
|
||||
log_queue = asyncio.Queue()
|
||||
loop = asyncio.get_running_loop()
|
||||
"""创建执行线程;从工作线程调用时必须显式传入主事件循环和日志队列。"""
|
||||
loop = loop or asyncio.get_running_loop()
|
||||
log_queue = log_queue or asyncio.Queue()
|
||||
thread_db = SessionLocal()
|
||||
runner = DouyuBatchRunner(
|
||||
db=thread_db,
|
||||
|
||||
@@ -6,7 +6,7 @@ import asyncio
|
||||
import threading
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -59,6 +59,38 @@ DOUYU_PROXY_TASK_TYPES = {
|
||||
}
|
||||
|
||||
|
||||
_DOUYU_RESULT_DEBUG_KEYS = {
|
||||
"raw",
|
||||
"query_params",
|
||||
"qq_app_scheme",
|
||||
"role_param_url",
|
||||
"game_role_bind_url",
|
||||
}
|
||||
|
||||
|
||||
def slim_douyu_task_result_for_storage(result: dict | None) -> dict | None:
|
||||
"""去掉任务结果里的调试大字段,避免每次轮询重复写大 JSON。
|
||||
|
||||
运行器在内存中仍使用完整 result;这里只影响落库和 WS 展示。二维码 URL、
|
||||
角色状态、积分、订单号等业务字段全部保留。
|
||||
"""
|
||||
if result is None:
|
||||
return None
|
||||
|
||||
def clean(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: clean(item)
|
||||
for key, item in value.items()
|
||||
if key not in _DOUYU_RESULT_DEBUG_KEYS
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [clean(item) for item in value]
|
||||
return value
|
||||
|
||||
return clean(result)
|
||||
|
||||
|
||||
class DouyuBatchRunnerCore:
|
||||
"""斗鱼任务执行器公共基础:批次状态、日志、任务落库与客户端构造。"""
|
||||
|
||||
@@ -196,9 +228,7 @@ class DouyuBatchRunnerCore:
|
||||
|
||||
sync_task_from_binding_runner(db, douyu_task_id=int(task.id))
|
||||
except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果
|
||||
logger.exception(
|
||||
"[douyu] 同步直播账号履约状态失败: task_id={}", task.id
|
||||
)
|
||||
logger.exception("[douyu] 同步直播账号履约状态失败: task_id={}", task.id)
|
||||
|
||||
def _sync_platform_fulfillment_operation(
|
||||
self,
|
||||
@@ -220,9 +250,7 @@ class DouyuBatchRunnerCore:
|
||||
message=message,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - 履约时间线不能覆盖任务执行结果
|
||||
logger.exception(
|
||||
"[douyu] 写入履约操作时间线失败: task_id={}", task.id
|
||||
)
|
||||
logger.exception("[douyu] 写入履约操作时间线失败: task_id={}", task.id)
|
||||
|
||||
def _mark_task(
|
||||
self,
|
||||
@@ -235,30 +263,28 @@ class DouyuBatchRunnerCore:
|
||||
task.status = status
|
||||
task.message = message[:512]
|
||||
if result is not None:
|
||||
task.result = result
|
||||
task.result = slim_douyu_task_result_for_storage(result)
|
||||
task.finished_at = datetime.now(UTC)
|
||||
if task.account_id and status in {"success", "failed", "error", "stopped"}:
|
||||
from .business_events import record_douyu_task_result
|
||||
|
||||
account = (
|
||||
db.query(Account).filter(Account.id == task.account_id).first()
|
||||
)
|
||||
if account is not None:
|
||||
record_douyu_task_result(
|
||||
db,
|
||||
account,
|
||||
task_id=int(task.id),
|
||||
task_type=str(task.task_type or ""),
|
||||
task_label=SUPPORTED_DOUYU_TASK_TYPES.get(
|
||||
str(task.task_type or ""),
|
||||
str(task.task_type or "斗鱼任务"),
|
||||
),
|
||||
scope=str(task.handbook_scope or ""),
|
||||
status=status,
|
||||
message=message,
|
||||
occurred_at=task.finished_at,
|
||||
actor_id=int(task.created_by),
|
||||
)
|
||||
account = (
|
||||
db.query(Account).filter(Account.id == task.account_id).first()
|
||||
if task.account_id and status in {"success", "failed", "error", "stopped"}
|
||||
else None
|
||||
)
|
||||
event_context = (
|
||||
{
|
||||
"task_id": int(task.id),
|
||||
"task_type": str(task.task_type or ""),
|
||||
"task_label": SUPPORTED_DOUYU_TASK_TYPES.get(
|
||||
str(task.task_type or ""),
|
||||
str(task.task_type or "斗鱼任务"),
|
||||
),
|
||||
"scope": str(task.handbook_scope or ""),
|
||||
"finished_at": task.finished_at,
|
||||
"created_by": int(task.created_by),
|
||||
}
|
||||
if account is not None
|
||||
else None
|
||||
)
|
||||
self._sync_platform_fulfillment_operation(
|
||||
db,
|
||||
task,
|
||||
@@ -267,6 +293,28 @@ class DouyuBatchRunnerCore:
|
||||
)
|
||||
db.commit()
|
||||
self._push_task_event(task)
|
||||
if account is not None and event_context is not None:
|
||||
try:
|
||||
from .business_events import record_douyu_task_result
|
||||
|
||||
record_douyu_task_result(
|
||||
db,
|
||||
account,
|
||||
task_id=event_context["task_id"],
|
||||
task_type=event_context["task_type"],
|
||||
task_label=event_context["task_label"],
|
||||
scope=event_context["scope"],
|
||||
status=status,
|
||||
message=message,
|
||||
occurred_at=event_context["finished_at"],
|
||||
actor_id=event_context["created_by"],
|
||||
)
|
||||
db.commit()
|
||||
except Exception: # noqa: BLE001 - 时间线旁路不能覆盖已提交的任务结果
|
||||
db.rollback()
|
||||
logger.exception(
|
||||
"[douyu] 写入账号任务结果事件失败: task_id={}", task.id
|
||||
)
|
||||
self._sync_platform_fulfillment(db, task)
|
||||
|
||||
def _update_task_progress(
|
||||
@@ -280,7 +328,7 @@ class DouyuBatchRunnerCore:
|
||||
task.status = status
|
||||
task.message = message[:512]
|
||||
if result is not None:
|
||||
task.result = result
|
||||
task.result = slim_douyu_task_result_for_storage(result)
|
||||
db.commit()
|
||||
self._push_task_event(task)
|
||||
self._sync_platform_fulfillment(db, task)
|
||||
|
||||
@@ -2,9 +2,11 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import and_, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -146,10 +148,42 @@ DOUYU_ACTIVE_PAYMENT_TASK_TYPES = (
|
||||
)
|
||||
|
||||
|
||||
def _env_positive_int(name: str, default: int) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
return value if value > 0 else default
|
||||
|
||||
|
||||
DOUYU_MAX_ACTIVE_TASKS = _env_positive_int("DOUYU_MAX_ACTIVE_TASKS", 250)
|
||||
|
||||
|
||||
class DouyuActivePaymentError(ValueError):
|
||||
"""同一账号已有未结束的充值任务。"""
|
||||
|
||||
|
||||
class DouyuTaskBackpressureError(ValueError):
|
||||
"""全局活跃斗鱼任务超过安全上限。"""
|
||||
|
||||
|
||||
def _ensure_active_task_budget(db: Session, additional: int) -> None:
|
||||
"""为任务创建提供全局背压,避免无限线程和连接同时挤入数据库。"""
|
||||
if additional <= 0:
|
||||
return
|
||||
active = (
|
||||
db.query(func.count(DouyuTask.id))
|
||||
.filter(DouyuTask.status.in_(DOUYU_ACTIVE_TASK_STATUSES))
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
if active + additional > DOUYU_MAX_ACTIVE_TASKS:
|
||||
raise DouyuTaskBackpressureError(
|
||||
f"斗鱼任务繁忙:当前活跃 {active} 个,"
|
||||
f"上限 {DOUYU_MAX_ACTIVE_TASKS} 个,请稍后重试"
|
||||
)
|
||||
|
||||
|
||||
def douyu_config_value(field: str, value):
|
||||
"""读取配置值;空值自动回退到默认值。"""
|
||||
default = DOUYU_CONFIG_DEFAULTS[field]
|
||||
@@ -348,6 +382,7 @@ def create_douyu_planned_tasks(
|
||||
raise ValueError("该任务不属于当前工作台")
|
||||
|
||||
accounts = visible_douyu_task_accounts(db, account_ids)
|
||||
_ensure_active_task_budget(db, len(accounts))
|
||||
_lock_payment_accounts(db, accounts, task_type)
|
||||
if (
|
||||
task_type in {"refresh_goods", "refresh_esports_goods", "refresh_xpd_goods"}
|
||||
@@ -375,33 +410,39 @@ def create_douyu_planned_tasks(
|
||||
db.add(task)
|
||||
task_rows.append((task, account))
|
||||
db.flush()
|
||||
events: list[dict] = []
|
||||
if task_rows:
|
||||
label = SUPPORTED_DOUYU_TASK_TYPES[task_type]
|
||||
emit_bulk(
|
||||
db,
|
||||
[
|
||||
{
|
||||
"account_id": account.id,
|
||||
"event_type": EVENT_TASK_STARTED,
|
||||
"title": f"{label}:任务已创建",
|
||||
"category": CATEGORY_ACTIVITY,
|
||||
"detail": {
|
||||
"task_id": task.id,
|
||||
"task_type": task_type,
|
||||
"scope": handbook_scope,
|
||||
"status": "planned",
|
||||
},
|
||||
"source": "douyu_task",
|
||||
"subject_type": "task",
|
||||
"subject_key": str(task.id),
|
||||
"actor_type": "user",
|
||||
"actor_id": created_by,
|
||||
"source_ref": f"douyu_task_started:{task.id}",
|
||||
}
|
||||
for task, account in task_rows
|
||||
],
|
||||
)
|
||||
events = [
|
||||
{
|
||||
"account_id": account.id,
|
||||
"event_type": EVENT_TASK_STARTED,
|
||||
"title": f"{label}:任务已创建",
|
||||
"category": CATEGORY_ACTIVITY,
|
||||
"detail": {
|
||||
"task_id": task.id,
|
||||
"task_type": task_type,
|
||||
"scope": handbook_scope,
|
||||
"status": "planned",
|
||||
},
|
||||
"source": "douyu_task",
|
||||
"subject_type": "task",
|
||||
"subject_key": str(task.id),
|
||||
"actor_type": "user",
|
||||
"actor_id": created_by,
|
||||
"source_ref": f"douyu_task_started:{task.id}",
|
||||
}
|
||||
for task, account in task_rows
|
||||
]
|
||||
# 任务行必须先独立提交。账号事件是旁路,不能在持有任务行锁时批量插入。
|
||||
db.commit()
|
||||
if events:
|
||||
try:
|
||||
emit_bulk(db, events)
|
||||
db.commit()
|
||||
except Exception: # noqa: BLE001 - 时间线不能回滚已经提交的任务
|
||||
db.rollback()
|
||||
logger.exception("批量写入斗鱼任务创建事件失败")
|
||||
return batch_id, len(accounts)
|
||||
|
||||
|
||||
@@ -424,39 +465,73 @@ def cleanup_orphan_douyu_tasks(
|
||||
if not tasks:
|
||||
return 0
|
||||
now = datetime.now(UTC)
|
||||
from .business_events import record_douyu_task_result
|
||||
|
||||
account_ids = {int(task.account_id) for task in tasks if task.account_id}
|
||||
accounts = (
|
||||
{
|
||||
account.id: account
|
||||
for account in db.query(Account).filter(Account.id.in_(account_ids)).all()
|
||||
}
|
||||
if account_ids
|
||||
else {}
|
||||
)
|
||||
event_contexts: list[tuple[dict, Account]] = []
|
||||
platform_task_ids: list[int] = []
|
||||
for task in tasks:
|
||||
task.status = "stopped"
|
||||
task.message = message
|
||||
task.finished_at = now
|
||||
if task.account_id:
|
||||
account = db.query(Account).filter(Account.id == task.account_id).first()
|
||||
if account is not None:
|
||||
record_douyu_task_result(
|
||||
db,
|
||||
account = accounts.get(int(task.account_id)) if task.account_id else None
|
||||
if account is not None:
|
||||
event_contexts.append(
|
||||
(
|
||||
{
|
||||
"task_id": int(task.id),
|
||||
"task_type": str(task.task_type or ""),
|
||||
"task_label": SUPPORTED_DOUYU_TASK_TYPES.get(
|
||||
str(task.task_type or ""),
|
||||
str(task.task_type or "斗鱼任务"),
|
||||
),
|
||||
"scope": str(task.handbook_scope or ""),
|
||||
},
|
||||
account,
|
||||
task_id=int(task.id),
|
||||
task_type=str(task.task_type or ""),
|
||||
task_label=SUPPORTED_DOUYU_TASK_TYPES.get(
|
||||
str(task.task_type or ""),
|
||||
str(task.task_type or "斗鱼任务"),
|
||||
),
|
||||
scope=str(task.handbook_scope or ""),
|
||||
status="stopped",
|
||||
message=message,
|
||||
occurred_at=now,
|
||||
)
|
||||
)
|
||||
if task.platform_task_id:
|
||||
from .platform_fulfillment.service import record_douyu_operation_result
|
||||
platform_task_ids.append(int(task.id))
|
||||
# 先提交任务终态,再写账号时间线,避免 account_events 与 douyu_tasks
|
||||
# 在并发创建和结束任务时形成反向锁依赖。
|
||||
db.commit()
|
||||
|
||||
from .business_events import record_douyu_task_result
|
||||
|
||||
for context, account in event_contexts:
|
||||
record_douyu_task_result(
|
||||
db,
|
||||
account,
|
||||
task_id=context["task_id"],
|
||||
task_type=context["task_type"],
|
||||
task_label=context["task_label"],
|
||||
scope=context["scope"],
|
||||
status="stopped",
|
||||
message=message,
|
||||
occurred_at=now,
|
||||
)
|
||||
db.commit()
|
||||
|
||||
if platform_task_ids:
|
||||
from .platform_fulfillment.service import record_douyu_operation_result
|
||||
|
||||
for task_id in platform_task_ids:
|
||||
task = db.get(DouyuTask, task_id)
|
||||
if task is None:
|
||||
continue
|
||||
record_douyu_operation_result(
|
||||
db,
|
||||
task,
|
||||
status="stopped",
|
||||
message=message,
|
||||
)
|
||||
db.commit()
|
||||
db.commit()
|
||||
return len(tasks)
|
||||
|
||||
|
||||
|
||||
@@ -138,7 +138,10 @@ def _mark_attempt_failed(
|
||||
失败重试会新建记录或新 attempt,因此是新的一条 —— 这是对的,那是新的事实。
|
||||
"""
|
||||
completed_at = datetime.now(UTC)
|
||||
attempt_at = task.attempt_at or completed_at
|
||||
# MySQL DATETIME(0) 会按毫秒四舍五入到整秒。若 source_ref 使用内存里
|
||||
# 带微秒的原值,第一次写 13:57:13.999 会落成 13:57:14;第二次重读后
|
||||
# 幂等键变成下一秒,唯一索引就无法拦住重复事件。先截断到整秒再统一使用。
|
||||
attempt_at = task.attempt_at or completed_at.replace(microsecond=0)
|
||||
attempt_utc = (
|
||||
attempt_at.replace(tzinfo=UTC)
|
||||
if attempt_at.tzinfo is None
|
||||
|
||||
Reference in New Issue
Block a user