perf(douyu): 限制任务并发并释放长事务连接
This commit is contained in:
@@ -24,6 +24,7 @@ from ..models import (
|
||||
ProxyConfig as ProxyConfigModel,
|
||||
)
|
||||
from .douyu_account_probe import activity_client
|
||||
from .douyu_concurrency import clamp_batch_task_concurrency
|
||||
from .douyu_goods_service import upsert_goods_snapshot
|
||||
from .douyu_service import (
|
||||
DOUYU_CONFIG_FIELDS,
|
||||
@@ -113,7 +114,7 @@ class DouyuBatchRunnerCore:
|
||||
self.payload = payload or {}
|
||||
self.log_queue = log_queue
|
||||
self.loop = loop
|
||||
self.concurrency = max(1, min(concurrency, 10))
|
||||
self.concurrency = clamp_batch_task_concurrency(concurrency)
|
||||
self._stop = threading.Event()
|
||||
self._counter_lock = threading.Lock()
|
||||
self._started = 0
|
||||
@@ -222,14 +223,25 @@ class DouyuBatchRunnerCore:
|
||||
}
|
||||
asyncio.run_coroutine_threadsafe(self.log_queue.put(event), self.loop)
|
||||
|
||||
def _sync_platform_fulfillment(self, db: Session, task: DouyuTask) -> None:
|
||||
def _sync_platform_fulfillment(self, douyu_task_id: int, *, enabled: bool) -> None:
|
||||
"""绑定/确认任务完成后立即刷新履约投影并写 Outbox。"""
|
||||
if not enabled:
|
||||
return
|
||||
try:
|
||||
from ..database import SessionLocal
|
||||
from .platform_fulfillment.service import sync_task_from_binding_runner
|
||||
|
||||
sync_task_from_binding_runner(db, douyu_task_id=int(task.id))
|
||||
# 当前批次会话刚刚提交过任务结果。履约投影使用独立短会话,避免
|
||||
# get_task_detail 留下的只读事务在下一轮外部 HTTP 轮询期间占连接。
|
||||
sync_db = SessionLocal()
|
||||
try:
|
||||
sync_task_from_binding_runner(sync_db, douyu_task_id=int(douyu_task_id))
|
||||
finally:
|
||||
sync_db.close()
|
||||
except Exception: # noqa: BLE001 - 履约投影失败不能覆盖任务执行结果
|
||||
logger.exception("[douyu] 同步直播账号履约状态失败: task_id={}", task.id)
|
||||
logger.exception(
|
||||
"[douyu] 同步直播账号履约状态失败: task_id={}", douyu_task_id
|
||||
)
|
||||
|
||||
def _sync_platform_fulfillment_operation(
|
||||
self,
|
||||
@@ -261,6 +273,8 @@ class DouyuBatchRunnerCore:
|
||||
message: str,
|
||||
result: dict | None = None,
|
||||
) -> None:
|
||||
task_id = int(task.id)
|
||||
has_platform_task = task.platform_task_id is not None
|
||||
task.status = status
|
||||
task.message = message[:512]
|
||||
if result is not None:
|
||||
@@ -316,7 +330,7 @@ class DouyuBatchRunnerCore:
|
||||
logger.exception(
|
||||
"[douyu] 写入账号任务结果事件失败: task_id={}", task.id
|
||||
)
|
||||
self._sync_platform_fulfillment(db, task)
|
||||
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
|
||||
|
||||
def _update_task_progress(
|
||||
self,
|
||||
@@ -326,13 +340,15 @@ class DouyuBatchRunnerCore:
|
||||
message: str,
|
||||
result: dict | None = None,
|
||||
) -> None:
|
||||
task_id = int(task.id)
|
||||
has_platform_task = task.platform_task_id is not None
|
||||
task.status = status
|
||||
task.message = message[:512]
|
||||
if result is not None:
|
||||
task.result = slim_douyu_task_result_for_storage(result)
|
||||
db.commit()
|
||||
self._push_task_event(task)
|
||||
self._sync_platform_fulfillment(db, task)
|
||||
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
|
||||
|
||||
def _upsert_goods(self, db: Session, goods: list[dict]) -> None:
|
||||
upsert_goods_snapshot(db, goods)
|
||||
|
||||
Reference in New Issue
Block a user