perf(douyu): 限制任务并发并释放长事务连接

This commit is contained in:
yml
2026-11-20 20:16:53 +08:00
parent 5d44ab4ca1
commit 18fd9840f7
16 changed files with 331 additions and 27 deletions
+22 -6
View File
@@ -24,6 +24,7 @@ from ..models import (
ProxyConfig as ProxyConfigModel,
)
from .douyu_account_probe import activity_client
from .douyu_concurrency import clamp_batch_task_concurrency
from .douyu_goods_service import upsert_goods_snapshot
from .douyu_service import (
DOUYU_CONFIG_FIELDS,
@@ -113,7 +114,7 @@ class DouyuBatchRunnerCore:
self.payload = payload or {}
self.log_queue = log_queue
self.loop = loop
self.concurrency = max(1, min(concurrency, 10))
self.concurrency = clamp_batch_task_concurrency(concurrency)
self._stop = threading.Event()
self._counter_lock = threading.Lock()
self._started = 0
@@ -222,14 +223,25 @@ class DouyuBatchRunnerCore:
}
asyncio.run_coroutine_threadsafe(self.log_queue.put(event), self.loop)
def _sync_platform_fulfillment(self, db: Session, task: DouyuTask) -> None:
def _sync_platform_fulfillment(self, douyu_task_id: int, *, enabled: bool) -> None:
"""绑定/确认任务完成后立即刷新履约投影并写 Outbox。"""
if not enabled:
return
try:
from ..database import SessionLocal
from .platform_fulfillment.service import sync_task_from_binding_runner
sync_task_from_binding_runner(db, douyu_task_id=int(task.id))
# 当前批次会话刚刚提交过任务结果。履约投影使用独立短会话,避免
# get_task_detail 留下的只读事务在下一轮外部 HTTP 轮询期间占连接。
sync_db = SessionLocal()
try:
sync_task_from_binding_runner(sync_db, douyu_task_id=int(douyu_task_id))
finally:
sync_db.close()
except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果
logger.exception("[douyu] 同步直播账号履约状态失败: task_id={}", task.id)
logger.exception(
"[douyu] 同步直播账号履约状态失败: task_id={}", douyu_task_id
)
def _sync_platform_fulfillment_operation(
self,
@@ -261,6 +273,8 @@ class DouyuBatchRunnerCore:
message: str,
result: dict | None = None,
) -> None:
task_id = int(task.id)
has_platform_task = task.platform_task_id is not None
task.status = status
task.message = message[:512]
if result is not None:
@@ -316,7 +330,7 @@ class DouyuBatchRunnerCore:
logger.exception(
"[douyu] 写入账号任务结果事件失败: task_id={}", task.id
)
self._sync_platform_fulfillment(db, task)
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
def _update_task_progress(
self,
@@ -326,13 +340,15 @@ class DouyuBatchRunnerCore:
message: str,
result: dict | None = None,
) -> None:
task_id = int(task.id)
has_platform_task = task.platform_task_id is not None
task.status = status
task.message = message[:512]
if result is not None:
task.result = slim_douyu_task_result_for_storage(result)
db.commit()
self._push_task_event(task)
self._sync_platform_fulfillment(db, task)
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
def _upsert_goods(self, db: Session, goods: list[dict]) -> None:
upsert_goods_snapshot(db, goods)