perf(douyu): 拆分任务槽并改造绑定扫码调度

This commit is contained in:
yml
2026-11-20 22:03:20 +08:00
parent 8869e427d2
commit 071d45ad1d
40 changed files with 1635 additions and 245 deletions
+39
View File
@@ -71,6 +71,21 @@ if not DATABASE_URL.startswith("sqlite"):
engine = create_engine(DATABASE_URL, **engine_options)
worker_engine_options = {
"connect_args": connect_args,
"echo": False,
}
if not DATABASE_URL.startswith("sqlite"):
worker_engine_options.update(
pool_pre_ping=True,
pool_size=_env_int("WORKER_DB_POOL_SIZE", 12),
max_overflow=_env_int("WORKER_DB_MAX_OVERFLOW", 4),
pool_timeout=_env_int("WORKER_DB_POOL_TIMEOUT", 30),
pool_recycle=_env_int("DB_POOL_RECYCLE", 1800),
)
worker_engine = create_engine(DATABASE_URL, **worker_engine_options)
if DATABASE_URL.startswith("sqlite"):
@@ -82,6 +97,14 @@ if DATABASE_URL.startswith("sqlite"):
cursor.execute("PRAGMA busy_timeout=30000")
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
@event.listens_for(worker_engine, "connect")
def _set_worker_sqlite_pragmas(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA journal_mode=WAL")
cursor.execute("PRAGMA busy_timeout=30000")
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
else:
@event.listens_for(engine, "connect")
@@ -96,8 +119,24 @@ else:
finally:
cursor.close()
@event.listens_for(worker_engine, "connect")
def _set_worker_mysql_session_guards(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
try:
cursor.execute(
"SET SESSION innodb_lock_wait_timeout = "
f"{_env_int('DB_LOCK_WAIT_TIMEOUT', 10)}"
)
finally:
cursor.close()
SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False)
WorkerSessionLocal = sessionmaker(
bind=worker_engine,
autocommit=False,
autoflush=False,
)
Base = declarative_base()
+8
View File
@@ -101,6 +101,10 @@ async def lifespan(app: FastAPI):
register_default_jobs()
start_scheduler()
from .services.binding_poll_scheduler import (
start_binding_poll_scheduler,
stop_binding_poll_scheduler,
)
from .services.integration import (
start_integration_outbox_worker,
stop_integration_outbox_worker,
@@ -113,17 +117,21 @@ async def lifespan(app: FastAPI):
start_platform_fulfillment_reconciler,
stop_platform_fulfillment_reconciler,
)
from .services.task_runtime import shutdown as shutdown_task_runtime
start_integration_outbox_worker()
start_platform_fulfillment_reconciler()
start_platform_fulfillment_auto_allocator()
start_binding_poll_scheduler()
try:
yield
finally:
stop_binding_poll_scheduler()
stop_platform_fulfillment_auto_allocator()
stop_platform_fulfillment_reconciler()
stop_integration_outbox_worker()
stop_scheduler()
shutdown_task_runtime(wait=False)
app = FastAPI(
@@ -0,0 +1,145 @@
"""add durable scheduling fields to douyu_tasks
Long waits such as QR scan polling must survive a process restart without
holding a worker thread. The scheduler claims due rows by lane and next_run_at.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision: str = "20261120_0101"
down_revision: str | None = "20261120_0100"
branch_labels: str | None = None
depends_on: str | None = None
INDEX_NAME = "ix_douyu_tasks_lane_due"
def _columns() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("douyu_tasks"):
return set()
return {column["name"] for column in inspector.get_columns("douyu_tasks")}
def _indexes() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("douyu_tasks"):
return set()
return {index["name"] for index in inspector.get_indexes("douyu_tasks")}
def upgrade() -> None:
columns = _columns()
if "queue_lane" not in columns:
op.add_column(
"douyu_tasks",
sa.Column(
"queue_lane",
sa.String(length=32),
nullable=False,
server_default="",
),
)
if "next_run_at" not in columns:
op.add_column(
"douyu_tasks",
sa.Column("next_run_at", sa.DateTime(), nullable=True),
)
if "run_deadline_at" not in columns:
op.add_column(
"douyu_tasks",
sa.Column("run_deadline_at", sa.DateTime(), nullable=True),
)
if "last_run_at" not in columns:
op.add_column(
"douyu_tasks",
sa.Column("last_run_at", sa.DateTime(), nullable=True),
)
if "attempt_count" not in columns:
op.add_column(
"douyu_tasks",
sa.Column(
"attempt_count",
sa.Integer(),
nullable=False,
server_default="0",
),
)
if "active_binding_key" not in columns:
op.add_column(
"douyu_tasks",
sa.Column(
"active_binding_key",
sa.String(length=64),
sa.Computed(
"CASE WHEN task_type = 'get_bind_qr' "
"AND platform_task_id IS NOT NULL "
"AND status IN ('planned', 'pending', 'running') "
"THEN CONCAT('bind:', platform_task_id) ELSE NULL END",
persisted=True,
),
nullable=True,
),
)
# Keep the newest active QR task if legacy concurrency produced duplicates.
# Older runtime rows are superseded; historical executed tasks are untouched.
op.execute(
sa.text(
"""
UPDATE douyu_tasks AS task
JOIN (
SELECT platform_task_id, MAX(id) AS keep_id
FROM douyu_tasks
WHERE task_type = 'get_bind_qr'
AND platform_task_id IS NOT NULL
AND status IN ('planned', 'pending', 'running')
GROUP BY platform_task_id
HAVING COUNT(*) > 1
) AS duplicate
ON duplicate.platform_task_id = task.platform_task_id
SET task.status = 'stopped',
task.message = '已被新的绑定二维码任务取代(迁移收口)',
task.finished_at = UTC_TIMESTAMP(),
task.next_run_at = NULL
WHERE task.task_type = 'get_bind_qr'
AND task.status IN ('planned', 'pending', 'running')
AND task.id <> duplicate.keep_id
"""
)
)
if INDEX_NAME not in _indexes():
op.create_index(
INDEX_NAME,
"douyu_tasks",
["queue_lane", "status", "next_run_at", "id"],
unique=False,
)
indexes = _indexes()
if "uq_douyu_tasks_active_binding" not in indexes:
op.create_index(
"uq_douyu_tasks_active_binding",
"douyu_tasks",
["active_binding_key"],
unique=True,
)
def downgrade() -> None:
if "uq_douyu_tasks_active_binding" in _indexes():
op.drop_index("uq_douyu_tasks_active_binding", table_name="douyu_tasks")
if INDEX_NAME in _indexes():
op.drop_index(INDEX_NAME, table_name="douyu_tasks")
columns = _columns()
for name in (
"active_binding_key",
"attempt_count",
"last_run_at",
"run_deadline_at",
"next_run_at",
"queue_lane",
):
if name in columns:
op.drop_column("douyu_tasks", name)
+37
View File
@@ -360,6 +360,18 @@ class DouyuTask(Base):
UTCDateTime, default=_utcnow, index=True
)
finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True)
# Durable background scheduling. Waiting work is not an active worker.
queue_lane: Mapped[str] = mapped_column(
String(32), nullable=False, default="", server_default=""
)
next_run_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True)
run_deadline_at: Mapped[datetime | None] = mapped_column(
UTCDateTime, nullable=True
)
last_run_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True)
attempt_count: Mapped[int] = mapped_column(
Integer, nullable=False, default=0, server_default="0"
)
platform_task_id: Mapped[int | None] = mapped_column(
BigInteger,
nullable=True,
@@ -369,6 +381,19 @@ class DouyuTask(Base):
account_assigned_to: Mapped[int | None] = mapped_column(
ForeignKey("users.id"), nullable=True
)
# One active QR task per fulfillment session, enforced by MySQL even when
# two application workers race through the row lock boundary.
active_binding_key: Mapped[str | None] = mapped_column(
String(64),
Computed(
"CASE WHEN task_type = 'get_bind_qr' "
"AND platform_task_id IS NOT NULL "
"AND status IN ('planned', 'pending', 'running') "
"THEN CONCAT('bind:', platform_task_id) ELSE NULL END",
persisted=True,
),
nullable=True,
)
__table_args__ = (
Index(
@@ -388,6 +413,18 @@ class DouyuTask(Base):
"handbook_scope",
"id",
),
Index(
"ix_douyu_tasks_lane_due",
"queue_lane",
"status",
"next_run_at",
"id",
),
Index(
"uq_douyu_tasks_active_binding",
"active_binding_key",
unique=True,
),
)
account: Mapped[Account] = relationship("Account", back_populates="douyu_tasks")
+12
View File
@@ -92,12 +92,24 @@ def list_jobs(
# 巡检与重登是两个作业,但通过数据库联动:巡检把行标成 invalid,重登作业的
# 候选条件正是这个字段。这里把「当前有多少条等着重登」一并返回,页面才能把
# 这层上下游关系显示出来,而不是让人以为两个开关互不相干。
from ..database import worker_engine
from ..services.automation.jobs.ck_relogin import count_pending_relogin
from ..services.binding_poll_scheduler import (
scheduler_status as binding_poll_status,
)
from ..services.task_runtime import lane_stats
pending_relogin = count_pending_relogin(db)
return {
"scheduler": scheduler_status(),
"runtime": {
"task_lanes": lane_stats(),
"worker_db_pool": getattr(worker_engine.pool, "status", lambda: "")()
if callable(getattr(worker_engine.pool, "status", None))
else "",
"binding_poll": binding_poll_status(),
},
"pending_relogin": pending_relogin,
"jobs": [
{
+2 -2
View File
@@ -27,7 +27,7 @@ from fastapi.responses import StreamingResponse
from sqlalchemy import case, func, or_
from sqlalchemy.orm import Session, defer, joinedload
from ..database import SessionLocal, get_db
from ..database import WorkerSessionLocal, get_db
from ..deps import get_current_user, require_permission
from ..models import Account, AuditLog, LoginTask, User
from ..models import ProxyConfig as ProxyConfigModel
@@ -649,7 +649,7 @@ def _start_relogin_accounts(
)
proxy = db.query(ProxyConfigModel).first()
thread_db = SessionLocal()
thread_db = WorkerSessionLocal()
max_login_retries, max_total_time = get_relogin_limits()
runner = LoginBatchRunner(
db=thread_db,
+2 -2
View File
@@ -28,7 +28,7 @@ from core.douyu import (
FishFinRechargeConfigError,
)
from ..database import SessionLocal, get_db
from ..database import WorkerSessionLocal, get_db
from ..deps import authenticate_websocket, get_current_user, require_permission
from ..models import (
Account,
@@ -1321,7 +1321,7 @@ async def ws_douyu_logs(websocket: WebSocket, batch_id: str):
if not user_has_permission(user, "douyu:task"):
await websocket.close(code=1008, reason="无权限")
return
db = SessionLocal()
db = WorkerSessionLocal()
try:
_require_batch_owner(db, user, batch_id)
except HTTPException:
+1 -1
View File
@@ -45,7 +45,7 @@ from core.huya.device_profile import (
)
from core.sms_provider import parse_sms_lines
from ...database import SessionLocal, get_db
from ...database import get_db
from ...deps import authenticate_websocket, get_current_user, require_permission
from ...models import (
HuyaAccount,
+3 -2
View File
@@ -5,6 +5,7 @@ from fastapi import APIRouter, Response
from core.huya.cookie_utils import cookie_value
from core.huya.http_client import HuyaHttpClient
from ...database import WorkerSessionLocal
from ...schemas import HuyaWorkbenchTransferToCollab
from ...services.huya_points_center import (
fetch_points_center_catalog,
@@ -339,7 +340,7 @@ async def create_task_batch(
log_queue = asyncio.Queue()
loop = asyncio.get_running_loop()
thread_db = SessionLocal()
thread_db = WorkerSessionLocal()
runner = HuyaBatchRunner(
db=thread_db,
batch_id=batch_id,
@@ -494,7 +495,7 @@ async def ws_huya_logs(websocket: WebSocket, batch_id: str):
if not _has_huya_perm(user, "huya:task"):
await websocket.close(code=1008, reason="无权限")
return
db = SessionLocal()
db = WorkerSessionLocal()
try:
_require_huya_batch_owner(db, user, batch_id)
except HTTPException:
+3 -3
View File
@@ -17,7 +17,7 @@ from fastapi.responses import StreamingResponse
from sqlalchemy import and_, func, or_
from sqlalchemy.orm import Session
from ..database import SessionLocal, get_db
from ..database import WorkerSessionLocal, get_db
from ..deps import authenticate_websocket, get_current_user, require_permission
from ..models import Account, LoginTask, User
from ..models import ProxyConfig as ProxyConfigModel
@@ -182,7 +182,7 @@ async def create_batch(
loop = asyncio.get_running_loop()
# 创建执行器(用独立的 DB 会话,因为在线程中运行)
thread_db = SessionLocal()
thread_db = WorkerSessionLocal()
runner = LoginBatchRunner(
db=thread_db,
account_ids=valid_ids,
@@ -687,7 +687,7 @@ async def ws_login_logs(websocket: WebSocket, batch_id: str):
# 归属校验:非 view_all 只能订阅自己创建的批次(与虎牙批次一致)。
if not user_has_permission(user, "login:view_all"):
check_db = SessionLocal()
check_db = WorkerSessionLocal()
try:
owned = (
check_db.query(LoginTask.id)
@@ -9,6 +9,7 @@ from hmac import compare_digest
from typing import Any
from fastapi import APIRouter, Depends, Header, HTTPException, Request
from fastapi.responses import JSONResponse
from sqlalchemy.orm import Session
from starlette.requests import ClientDisconnect
@@ -132,12 +133,16 @@ async def start_customer_binding(
task_id: int,
req: PlatformFulfillmentCustomerBindingAction,
db: Session = Depends(get_db),
) -> dict[str, Any]:
return await service.customer_start_binding(
) -> JSONResponse:
payload = await service.customer_start_binding(
db,
task_id=task_id,
force_refresh=req.force_refresh,
)
return JSONResponse(
status_code=202 if payload.get("accepted") else 200,
content=payload,
)
@router.get(
+26 -17
View File
@@ -6,7 +6,7 @@ import io
import threading
import uuid
import zipfile
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import as_completed
from dataclasses import dataclass, field
from datetime import UTC, datetime
from types import SimpleNamespace
@@ -18,10 +18,11 @@ from core.douyu import DouyuLogin, create_login_api_strategy
from core.douyu.login import AccountLike
from core.douyu.proxy_fetcher import ProxyFetcher
from ..database import SessionLocal
from ..database import WorkerSessionLocal
from ..models import ProxyConfig as ProxyConfigModel
from .account_events_service import CATEGORY_AUTH, emit
from .proxy_service import build_proxy_runtime
from .task_runtime import TaskLane, submit
def _now() -> datetime:
@@ -251,7 +252,7 @@ class AccountCheckRunner:
status = str(item.get("status") or "")
message = str(item.get("message") or "")
try:
db = SessionLocal()
db = WorkerSessionLocal()
try:
emit(
db,
@@ -422,21 +423,29 @@ class AccountCheckRunner:
if self._shared_proxy_fetcher:
self._shared_proxy_fetcher.warmup_whitelist()
with ThreadPoolExecutor(max_workers=self.batch.concurrency) as executor:
futures = []
for index, account in enumerate(self.accounts):
if self._stop.is_set():
self._set_item(
index,
status="stopped",
message="已停止",
finished_at=_now(),
)
continue
futures.append(executor.submit(self._run_one, index, account))
futures = []
for index, account in enumerate(self.accounts):
if self._stop.is_set():
self._set_item(
index,
status="stopped",
message="已停止",
finished_at=_now(),
)
continue
future = submit(
TaskLane.ACCOUNT_CHECK,
self._run_one,
index,
account,
stopped=self._stop.is_set,
)
if future is None:
break
futures.append(future)
for future in as_completed(futures):
future.result()
for future in as_completed(futures):
future.result()
except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
self._set_batch(
status="error", message=f"批次执行异常: {exc}", finished_at=_now()
@@ -9,7 +9,7 @@
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import as_completed
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -34,6 +34,7 @@ from ...douyu_account_probe import (
probe_session,
)
from ...douyu_service import cookie_holding_filter, cookie_operable_filter
from ...task_runtime import TaskLane, submit
JOB_KEY = "ck_health"
FLUSH_EVERY = 25
@@ -146,11 +147,14 @@ def run_ck_health(db: Session, job: AutomationJob) -> dict[str, Any]:
results: dict[int, dict[str, Any]] = {}
concurrency = max(min(int(job.concurrency or 1), 16), 1)
with ThreadPoolExecutor(max_workers=concurrency) as pool:
futures = {pool.submit(_detect, account): account for account in accounts}
for future in as_completed(futures):
account = futures[future]
results[account.id] = future.result()
futures = {}
for account in accounts:
future = submit(TaskLane.ACCOUNT_CHECK, _detect, account)
if future is not None:
futures[future] = account
for future in as_completed(futures):
account = futures[future]
results[account.id] = future.result()
# 只写检测结果三件套;cookie / status / attempt_* 一律不动。
valid = invalid = errors = 0
@@ -20,7 +20,7 @@ from typing import Any
from sqlalchemy import func, or_
from sqlalchemy.orm import Session
from ....database import SessionLocal
from ....database import WorkerSessionLocal
from ....models import Account, AutomationJob, LoginTask, User
from ....models import ProxyConfig as ProxyConfigModel
from ...account_events_service import EVENT_RELOGIN_FAILED, emit
@@ -148,7 +148,7 @@ def settle_fresh_results(
退避时间,其中 2 条还被标成「极验」进了待人工队列。SQLite 的读一致性与 MySQL
不同,这类问题在 SQLite 上复现不出来。
"""
session = SessionLocal()
session = WorkerSessionLocal()
try:
fresh, order = _load_batch_state(session, batch_id)
counters = settle_results(
@@ -241,7 +241,7 @@ def run_ck_relogin(db: Session, job: AutomationJob) -> dict[str, Any]:
_, max_total_time = get_relogin_limits()
# 批次用独立会话(runner 会在自己的线程/会话里写库),跑完再回来读结果。
runner_db = SessionLocal()
runner_db = WorkerSessionLocal()
runner = LoginBatchRunner(
db=runner_db,
account_ids=account_ids,
+6 -6
View File
@@ -24,7 +24,7 @@ from typing import Any
from loguru import logger
from sqlalchemy.orm import Session
from ...database import SessionLocal
from ...database import WorkerSessionLocal
from ...models import AutomationJob, AutomationRun
# 调度轮询间隔,也是作业的运行节奏:每个 tick 跑一轮。
@@ -71,7 +71,7 @@ _REGISTRY: dict[str, JobSpec] = {}
#: 同时最多几个作业在跑。作业之间互不依赖,并行是为了**慢作业不挡住快作业**
#: (以前串行:一个作业跑 4 分钟,CK 巡检就得等 4 分钟)。
_JOB_MAX_WORKERS = 4
_JOB_MAX_WORKERS = 2
#: 运行间隔的下限:太短会变成忙轮询,没有意义
MIN_RUN_INTERVAL_SECONDS = 30
@@ -190,7 +190,7 @@ def _run_job_isolated(job_key: str) -> dict[str, Any] | None:
多个作业共用一个 session 会互相踩事务(而他们现在是真的并行了)。
重新按 key 取行,是因为调度线程里的那行可能已经过期。
"""
db = SessionLocal()
db = WorkerSessionLocal()
try:
job = db.query(AutomationJob).filter(AutomationJob.key == job_key).first()
spec = _REGISTRY.get(job_key)
@@ -337,7 +337,7 @@ def _run_one(db: Session, job: AutomationJob, spec: JobSpec) -> dict[str, Any]:
db.rollback()
logger.exception("自动化作业 {} 提交状态失败,尝试独立连接回写", job.key)
try:
rescue_db = SessionLocal()
rescue_db = WorkerSessionLocal()
try:
r = (
rescue_db.query(AutomationRun)
@@ -412,7 +412,7 @@ def run_job_now(db: Session, key: str) -> dict[str, Any]:
def _loop() -> None:
db = SessionLocal()
db = WorkerSessionLocal()
try:
ensure_jobs(db)
except Exception: # noqa: BLE001 初始化失败不应让线程静默退出
@@ -424,7 +424,7 @@ def _loop() -> None:
"自动化调度器已启动:心跳 {}s,作业周期 {}s", TICK_SECONDS, JOB_INTERVAL_SECONDS
)
while not _stop_event.wait(TICK_SECONDS):
db = SessionLocal()
db = WorkerSessionLocal()
try:
# wait=False:派发完就返回 —— tick 不再被慢作业拖住
run_due_jobs(db, wait=False)
@@ -0,0 +1,212 @@
"""Database-backed scheduler for scan-wait binding tasks."""
from __future__ import annotations
import os
import threading
from concurrent.futures import Future
from datetime import UTC, datetime, timedelta
from loguru import logger
from sqlalchemy.orm import joinedload
from ..database import WorkerSessionLocal
from ..models import DouyuTask
from .douyu_runner import DouyuBatchRunner
from .task_runtime import TaskLane, submit
DEFAULT_INTERVAL_SECONDS = 1
DEFAULT_BATCH_SIZE = 20
CLAIM_SECONDS = 60
_stop_event = threading.Event()
_thread: threading.Thread | None = None
_lock = threading.Lock()
_in_flight: set[int] = set()
_in_flight_lock = threading.Lock()
def _env_int(name: str, default: int, minimum: int, maximum: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return max(minimum, min(maximum, value))
def _claim_due_binding_tasks(limit: int) -> list[int]:
db = WorkerSessionLocal()
now = datetime.now(UTC)
claimed: list[int] = []
try:
rows = (
db.query(DouyuTask)
.filter(
DouyuTask.task_type == "get_bind_qr",
DouyuTask.status == "running",
DouyuTask.queue_lane == "binding_poll",
DouyuTask.next_run_at.isnot(None),
DouyuTask.next_run_at <= now,
)
.order_by(DouyuTask.next_run_at.asc(), DouyuTask.id.asc())
.with_for_update(skip_locked=True)
.limit(limit)
.all()
)
for row in rows:
task_id = int(row.id)
with _in_flight_lock:
if task_id in _in_flight:
continue
_in_flight.add(task_id)
row.next_run_at = now + timedelta(seconds=CLAIM_SECONDS)
claimed.append(task_id)
db.commit()
return claimed
except Exception:
db.rollback()
with _in_flight_lock:
for task_id in claimed:
_in_flight.discard(task_id)
raise
finally:
db.close()
def poll_binding_task_once(task_id: int) -> None:
db = WorkerSessionLocal()
try:
task = (
db.query(DouyuTask)
.options(joinedload(DouyuTask.account))
.filter(DouyuTask.id == int(task_id))
.first()
)
if (
task is None
or task.status != "running"
or task.task_type != "get_bind_qr"
or task.queue_lane != "binding_poll"
):
return
account = task.account
if account is None or not (account.cookie or ""):
task.queue_lane = ""
task.next_run_at = None
runner = DouyuBatchRunner(
db,
task.batch_id,
"get_bind_qr",
close_db=False,
)
runner._mark_task(db, task, "failed", "账号 Cookie 已失效,无法继续检测绑定")
return
runner = DouyuBatchRunner(
db,
task.batch_id,
"get_bind_qr",
close_db=False,
)
config = runner._config_info(db)
runner.poll_bind_role_once(db, task, account, config)
except Exception as exc: # noqa: BLE001 - the claimed row is retried later
db.rollback()
logger.warning("绑定扫码定时检测失败 task_id={}: {}", task_id, exc)
finally:
db.close()
with _in_flight_lock:
_in_flight.discard(int(task_id))
def _release_callback(_future: Future) -> None:
try:
_future.result()
except Exception as exc: # noqa: BLE001 - worker already logs; keep scheduler alive
logger.warning("绑定扫码任务异常: {}", exc)
def run_binding_poll_dispatch_once(limit: int = DEFAULT_BATCH_SIZE) -> int:
claimed = _claim_due_binding_tasks(max(1, limit))
submitted = 0
for task_id in claimed:
future = submit(
TaskLane.BINDING_POLL,
poll_binding_task_once,
task_id,
stopped=_stop_event.is_set,
)
if future is None:
with _in_flight_lock:
_in_flight.discard(task_id)
break
future.add_done_callback(_release_callback)
submitted += 1
return submitted
def _run() -> None:
interval = _env_int(
"BINDING_POLL_DISPATCH_SECONDS",
DEFAULT_INTERVAL_SECONDS,
1,
30,
)
while not _stop_event.is_set():
try:
batch_size = _env_int(
"BINDING_POLL_DISPATCH_BATCH",
DEFAULT_BATCH_SIZE,
1,
200,
)
run_binding_poll_dispatch_once(batch_size)
except Exception as exc: # noqa: BLE001 - scheduler must survive one bad tick
logger.warning("绑定扫码调度轮次失败: {}", exc)
_stop_event.wait(interval)
def start_binding_poll_scheduler() -> bool:
global _thread
with _lock:
if _thread is not None and _thread.is_alive():
return False
_stop_event.clear()
_thread = threading.Thread(
target=_run,
name="binding-poll-scheduler",
daemon=True,
)
_thread.start()
logger.info("绑定扫码定时调度器已启动")
return True
def stop_binding_poll_scheduler() -> None:
global _thread
_stop_event.set()
with _lock:
thread = _thread
_thread = None
if thread is not None and thread.is_alive():
thread.join(timeout=3)
def scheduler_status() -> dict[str, object]:
with _lock:
running = _thread is not None and _thread.is_alive()
with _in_flight_lock:
in_flight = len(_in_flight)
return {
"running": running,
"in_flight": in_flight,
"claim_seconds": CLAIM_SECONDS,
}
__all__ = [
"poll_binding_task_once",
"run_binding_poll_dispatch_once",
"scheduler_status",
"start_binding_poll_scheduler",
"stop_binding_poll_scheduler",
]
+10 -5
View File
@@ -9,7 +9,7 @@ import asyncio
import os
import threading
from web.backend.database import SessionLocal
from web.backend.database import WorkerSessionLocal
from .douyu_concurrency import clamp_batch_task_concurrency
from .douyu_runner import DouyuBatchRunner, douyu_batch_registry
@@ -39,10 +39,15 @@ def start_douyu_batch(
loop: asyncio.AbstractEventLoop | None = None,
log_queue: asyncio.Queue | None = None,
) -> None:
"""创建执行线程;从工作线程调用时必须显式传入主事件循环和日志队列。"""
loop = loop or asyncio.get_running_loop()
log_queue = log_queue or asyncio.Queue()
thread_db = SessionLocal()
"""Create the batch coordinator; websocket logging is optional."""
if loop is None:
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if log_queue is None and loop is not None:
log_queue = asyncio.Queue()
thread_db = WorkerSessionLocal()
runner = DouyuBatchRunner(
db=thread_db,
batch_id=batch_id,
+3 -20
View File
@@ -1,7 +1,7 @@
"""斗鱼后台任务的进程级并发限制。
"""斗鱼批次级并发限制。
任务线程和数据库连接池共享同一个 Uvicorn 进程。这里把「每批并发」和
「全进程同时执行数」分开限制,避免多个批次叠加后把请求线程和连接池挤空。
Process-wide execution is owned by ``task_runtime`` lanes; this module only
keeps the per-batch cap and the overview sub-limit.
"""
from __future__ import annotations
@@ -23,16 +23,11 @@ DOUYU_BATCH_TASK_CONCURRENCY = _env_positive_int(
"DOUYU_BATCH_TASK_CONCURRENCY",
5,
)
DOUYU_MAX_RUNNING_TASKS = _env_positive_int(
"DOUYU_MAX_RUNNING_TASKS",
12,
)
DOUYU_MAX_ACTIVE_OVERVIEW_TASKS = _env_positive_int(
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
2,
)
_task_slots = threading.BoundedSemaphore(DOUYU_MAX_RUNNING_TASKS)
_overview_slots = threading.BoundedSemaphore(DOUYU_MAX_ACTIVE_OVERVIEW_TASKS)
@@ -47,15 +42,6 @@ def _acquire(slots: threading.BoundedSemaphore, stopped: Callable[[], bool]) ->
return False
def acquire_running_task_slot(stopped: Callable[[], bool]) -> bool:
"""等待一个全进程任务槽;停止时返回 False。"""
return _acquire(_task_slots, stopped)
def release_running_task_slot() -> None:
_task_slots.release()
def acquire_overview_slot(stopped: Callable[[], bool]) -> bool:
"""等待一个一键查询槽;该限制跨所有批次共享。"""
return _acquire(_overview_slots, stopped)
@@ -68,10 +54,7 @@ def release_overview_slot() -> None:
__all__ = [
"DOUYU_BATCH_TASK_CONCURRENCY",
"DOUYU_MAX_ACTIVE_OVERVIEW_TASKS",
"DOUYU_MAX_RUNNING_TASKS",
"acquire_overview_slot",
"acquire_running_task_slot",
"clamp_batch_task_concurrency",
"release_overview_slot",
"release_running_task_slot",
]
+50 -19
View File
@@ -2,21 +2,20 @@
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor, as_completed
import time
from concurrent.futures import as_completed
from sqlalchemy.orm import joinedload
from core.douyu import DouyuActivityError
from ..database import SessionLocal
from ..database import WorkerSessionLocal
from ..models import DouyuTask
from .cookie_check_service import check_douyu_cookie
from .douyu_account_probe import persist_probe_result, probe_session
from .douyu_concurrency import (
acquire_overview_slot,
acquire_running_task_slot,
release_overview_slot,
release_running_task_slot,
)
from .douyu_runner_bind import BindMixin
from .douyu_runner_core import ( # noqa: F401 (douyu_batch_registry 供 routers 重导出)
@@ -30,6 +29,11 @@ from .douyu_runner_manual import ManualMixin
from .douyu_runner_overview import OverviewMixin
from .douyu_runner_xpd import XpdMixin
from .douyu_service import update_account_profile_from_cookie
from .task_runtime import douyu_lane_for_task, submit
# Compatibility alias for tests and integrations that replace the worker
# session factory at the runner boundary.
SessionLocal = WorkerSessionLocal
class DouyuBatchRunner(
@@ -47,19 +51,13 @@ class DouyuBatchRunner(
def _execute_one(self, task_id: int, config: dict, total: int):
overview_task = self.task_type == "query_account_overview"
overview_slot = False
running_slot = False
if overview_task:
overview_slot = acquire_overview_slot(self._stop.is_set)
if not overview_slot:
return
try:
running_slot = acquire_running_task_slot(self._stop.is_set)
if not running_slot:
return
self._execute_one_with_slot(task_id, config, total)
finally:
if running_slot:
release_running_task_slot()
if overview_slot:
release_overview_slot()
@@ -221,19 +219,52 @@ class DouyuBatchRunner(
self._push_task_event(task)
total = len(tasks)
with ThreadPoolExecutor(max_workers=self.concurrency) as executor:
futures = []
for task in tasks:
if self._stop.is_set():
break
futures.append(
executor.submit(self._execute_one, task.id, config, total)
lane = douyu_lane_for_task(self.task_type)
pending: dict = {}
next_index = 0
while next_index < total or pending:
while (
not self._stop.is_set()
and next_index < total
and len(pending) < self.concurrency
):
task = tasks[next_index]
future = submit(
lane,
self._execute_one,
task.id,
config,
total,
stopped=self._stop.is_set,
)
for future in as_completed(futures):
if future is None:
break
pending[future] = (int(task.id), time.monotonic())
next_index += 1
if not pending:
break
try:
completed = list(as_completed(pending, timeout=0.5))
except TimeoutError:
continue
for future in completed:
task_id, submitted_at = pending.pop(future)
try:
future.result()
except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
self._push_log("error", f"Worker 异常: {exc}")
self._push_log(
"error",
f"Worker 异常 task_id={task_id}: {exc}",
)
else:
waited = time.monotonic() - submitted_at
if waited >= 5:
self._push_log(
"info",
f"任务 task_id={task_id} 在 {lane.value} 槽排队 {waited:.1f}s",
)
if self._stop.is_set():
self._push_log("warning", f"斗鱼任务批次 {self.batch_id} 已停止")
+180 -28
View File
@@ -632,6 +632,172 @@ class BindMixin:
)
return "", result
@staticmethod
def _schedule_next_bind_poll(
task: DouyuTask,
*,
now: datetime,
interval_seconds: int,
) -> None:
task.queue_lane = "binding_poll"
task.next_run_at = now + timedelta(seconds=max(1, interval_seconds))
task.last_run_at = now
task.attempt_count = int(task.attempt_count or 0) + 1
def poll_bind_role_once(
self,
db: Session,
task: DouyuTask,
account: Account,
config: dict,
) -> None:
"""Run one scheduled binding poll and immediately release the worker."""
now = datetime.now(UTC)
task.last_run_at = now
task.attempt_count = int(task.attempt_count or 0) + 1
result = dict(task.result or {})
query_aliases = [
str(value)
for value in (result.get("query_act_aliases") or [])
if str(value or "").strip()
]
if not query_aliases:
query_aliases = self._query_bind_act_aliases(config)
deadline = task.run_deadline_at
if deadline is not None and now >= deadline:
result.update(
{
"bind_polling": False,
"bind_ready_for_confirm": False,
"bind_phase": "role_timeout",
}
)
task.queue_lane = ""
task.next_run_at = None
self._mark_task(
db,
task,
"failed",
"绑定二维码已过期,未识别到新扫码角色",
result,
)
return
client = self._client(str(account.cookie or ""))
candidates = self._fetch_bind_info_candidates(client, query_aliases)
if not candidates:
attempts = max(1, int(task.attempt_count or 1))
backoff = min(30, max(DOUYU_BIND_ROLE_POLL_INTERVAL, attempts * 2))
result["bind_poll_error"] = "所有 actAlias 查询绑定信息失败"
result["bind_polling"] = True
result["bind_phase"] = "waiting_scan"
self._schedule_next_bind_poll(task, now=now, interval_seconds=backoff)
self._update_task_progress(
db,
task,
"running",
"等待扫码绑定:角色查询暂时失败",
result,
sync_platform=False,
)
return
baseline_role_name = str(result.get("baseline_role_name") or "")
baseline_is_bound_act = bool(result.get("baseline_is_bound_act"))
baseline_info = result.get("before_bind_info")
if not isinstance(baseline_info, dict):
baseline_info = None
bind_info = (
self._pick_bind_info(
candidates,
baseline_role_name=baseline_role_name,
baseline_is_bound_act=baseline_is_bound_act,
baseline_info=baseline_info,
prefer_pending=True,
prefer_aliases=query_aliases,
)
or candidates[0]
)
snapshot = self._bind_snapshot(bind_info)
is_pending_role = self._is_pending_role(
bind_info,
baseline_role_name=baseline_role_name,
baseline_is_bound_act=baseline_is_bound_act,
baseline_info=baseline_info,
)
query_alias = str(bind_info.get("act_alias") or "")
summary = (
f"act={query_alias or '-'} "
f"{self._format_bind_summary(bind_info, pending=is_pending_role)}"
)
if is_pending_role:
task.queue_lane = ""
task.next_run_at = None
result.update(
{
**snapshot,
"act_alias": query_alias,
"query_act_alias": query_alias,
"bind_ready_for_confirm": True,
"bind_confirmed": False,
"bind_phase": "role_ready",
"bind_polling": False,
"poll_count": int(result.get("poll_count") or 0) + 1,
"bind_summary": summary,
}
)
account.updated_at = now
self._mark_task(
db,
task,
"success",
f"已识别角色: {snapshot['role_name']},待确认绑定",
result,
)
return
if snapshot["role_name"] and snapshot["is_bound_act"]:
result["current_role_name"] = snapshot["role_name"]
result["current_area_name"] = snapshot["area_name"]
result["current_plat_name"] = snapshot["plat_name"]
result.update(
{
"bind_info": bind_info,
"act_alias": query_alias,
"query_act_alias": query_alias,
"is_bound_act": snapshot["is_bound_act"],
"is_bound_role": snapshot["is_bound_role"],
"is_bound_account": snapshot["is_bound_account"],
"need_bind_act": snapshot["need_bind_act"],
"need_bind_role": snapshot["need_bind_role"],
"change_role_wait_time": snapshot["change_role_wait_time"],
"can_change_role": snapshot["can_change_role"],
"role_name": "",
"area_name": "",
"plat_name": "",
"bind_ready_for_confirm": False,
"bind_confirmed": False,
"bind_phase": "waiting_scan",
"bind_polling": True,
"poll_count": int(result.get("poll_count") or 0) + 1,
"bind_summary": summary,
}
)
self._schedule_next_bind_poll(
task,
now=now,
interval_seconds=DOUYU_BIND_ROLE_POLL_INTERVAL,
)
self._update_task_progress(
db,
task,
"running",
f"等待扫码绑定 ({summary})",
result,
sync_platform=False,
)
def _execute_get_bind_qr(
self, db: Session, task: DouyuTask, account: Account, cookie: str, config: dict
):
@@ -763,40 +929,26 @@ class BindMixin:
"qr_expires_at": (
qr_generated_at + timedelta(seconds=DOUYU_BIND_ROLE_POLL_SECONDS)
).isoformat(),
"baseline_role_name": current_role_name,
"baseline_is_bound_act": before_snapshot["is_bound_act"],
"poll_count": 0,
}
account.updated_at = datetime.now(UTC)
# 关键:先把二维码 progress 出去,前端 running 期间即可弹窗扫码。
task.queue_lane = "binding_poll"
task.next_run_at = qr_generated_at + timedelta(
seconds=DOUYU_BIND_ROLE_POLL_INTERVAL
)
task.run_deadline_at = qr_generated_at + timedelta(
seconds=DOUYU_BIND_ROLE_POLL_SECONDS
)
task.last_run_at = qr_generated_at
task.attempt_count = 0
# Publish the QR immediately, then return without holding a worker while
# the customer scans. The durable scheduler resumes role polling.
self._update_task_progress(
db, task, "running", "已生成绑定二维码,等待扫码绑定", result
)
role_name, result = self._wait_bind_role_result(
client,
db,
task,
account,
query_aliases,
result,
baseline_role_name=current_role_name,
baseline_is_bound_act=before_snapshot["is_bound_act"],
baseline_info=before,
)
if role_name:
self._mark_task(
db, task, "success", f"已识别角色: {role_name},待确认绑定", result
)
return
if result.get("bind_phase") == "stopped":
self._mark_task(db, task, "stopped", "任务已停止", result)
return
last_summary = str(result.get("bind_summary") or "")
timeout_msg = "已生成绑定二维码,未检测到新扫码角色"
if current_role_name:
timeout_msg = f"{timeout_msg}(当前仍是 {current_role_name})"
if last_summary:
timeout_msg = f"{timeout_msg} | {last_summary}"
self._mark_task(db, task, "success", timeout_msg, result)
def _execute_confirm_bind(
self, db: Session, task: DouyuTask, account: Account, cookie: str, config: dict
):
+8 -5
View File
@@ -228,12 +228,12 @@ class DouyuBatchRunnerCore:
if not enabled:
return
try:
from ..database import SessionLocal
from ..database import WorkerSessionLocal
from .platform_fulfillment.service import sync_task_from_binding_runner
# 当前批次会话刚刚提交过任务结果。履约投影使用独立短会话,避免
# get_task_detail 留下的只读事务在下一轮外部 HTTP 轮询期间占连接。
sync_db = SessionLocal()
sync_db = WorkerSessionLocal()
try:
sync_task_from_binding_runner(sync_db, douyu_task_id=int(douyu_task_id))
finally:
@@ -339,6 +339,8 @@ class DouyuBatchRunnerCore:
status: str,
message: str,
result: dict | None = None,
*,
sync_platform: bool = True,
) -> None:
task_id = int(task.id)
has_platform_task = task.platform_task_id is not None
@@ -348,7 +350,8 @@ class DouyuBatchRunnerCore:
task.result = slim_douyu_task_result_for_storage(result)
db.commit()
self._push_task_event(task)
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
if sync_platform:
self._sync_platform_fulfillment(task_id, enabled=has_platform_task)
def _upsert_goods(self, db: Session, goods: list[dict]) -> None:
upsert_goods_snapshot(db, goods)
@@ -455,8 +458,8 @@ class DouyuBatchRegistry:
def register(
self,
batch_id: str,
log_queue: asyncio.Queue,
loop: asyncio.AbstractEventLoop,
log_queue: asyncio.Queue | None,
loop: asyncio.AbstractEventLoop | None,
runner: DouyuBatchRunner,
):
self._batches[batch_id] = {
+14 -2
View File
@@ -7,7 +7,7 @@ import uuid
from datetime import UTC, datetime
from loguru import logger
from sqlalchemy import and_, func, select
from sqlalchemy import and_, func, or_, select
from sqlalchemy.orm import Session
from core.douyu.cookie_utils import cookie_value
@@ -468,10 +468,22 @@ def cleanup_orphan_douyu_tasks(
elif active_batch_ids is not None and active_batch_ids:
query = query.filter(~DouyuTask.batch_id.in_(list(active_batch_ids)))
now = datetime.now(UTC)
# Scan waits are durable. A restarted scheduler can resume them from the
# database, so startup cleanup must not turn them into stopped tasks.
query = query.filter(
or_(
DouyuTask.task_type != "get_bind_qr",
DouyuTask.queue_lane != "binding_poll",
DouyuTask.next_run_at.is_(None),
DouyuTask.run_deadline_at.is_(None),
DouyuTask.run_deadline_at <= now,
)
)
tasks = query.all()
if not tasks:
return 0
now = datetime.now(UTC)
account_ids = {int(task.account_id) for task in tasks if task.account_id}
accounts = (
{
+36 -27
View File
@@ -4,7 +4,7 @@ from __future__ import annotations
import threading
import uuid
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import as_completed
from dataclasses import dataclass, field
from datetime import UTC, datetime
@@ -13,7 +13,7 @@ from core.huya.auto_register import HuyaAutoRegisterResult, register_huya_with_s
from core.huya.cookie_utils import normalize_huya_cookie
from core.sms_provider import SmsLine
from ..database import SessionLocal
from ..database import WorkerSessionLocal
from ..models import (
HuyaRegisterBatch as HuyaRegisterBatchModel,
)
@@ -28,6 +28,7 @@ from ..models import (
)
from .huya_service import upsert_huya_cookie
from .proxy_service import build_proxy_runtime
from .task_runtime import TaskLane, submit
# 运行中状态:服务中断后视为未完成,可继续
RUNNING_ITEM_STATUSES = frozenset({"sending", "waiting", "changing", "logging"})
@@ -209,7 +210,7 @@ def load_batch_snapshot(
batch_id: str, *, recover_interrupted: bool = True
) -> dict | None:
"""从数据库加载批次详情;若服务中断则标记为 interrupted。"""
db = SessionLocal()
db = WorkerSessionLocal()
try:
batch_row = (
db.query(HuyaRegisterBatchModel)
@@ -258,7 +259,7 @@ def list_batch_summaries(
) -> list[dict]:
"""列出最近的注册批次摘要。live_batch_ids 中的 running 保持运行中。"""
live = live_batch_ids or set()
db = SessionLocal()
db = WorkerSessionLocal()
try:
rows = (
db.query(HuyaRegisterBatchModel)
@@ -335,7 +336,7 @@ def export_success_logs_text(
limit: int = 5000,
) -> str:
"""从成功流水表导出 txt。"""
db = SessionLocal()
db = WorkerSessionLocal()
try:
query = db.query(HuyaRegisterSuccessLog).order_by(
HuyaRegisterSuccessLog.id.asc()
@@ -368,7 +369,7 @@ def list_success_logs(
limit: int = 200,
) -> list[dict]:
"""列出成功流水(含密码,供管理端展示/导出)。"""
db = SessionLocal()
db = WorkerSessionLocal()
try:
query = db.query(HuyaRegisterSuccessLog).order_by(
HuyaRegisterSuccessLog.id.desc()
@@ -450,7 +451,7 @@ class HuyaRegisterRunner:
"""把批次汇总写回数据库。"""
if not self.batch.db_id:
return
db = SessionLocal()
db = WorkerSessionLocal()
try:
row = (
db.query(HuyaRegisterBatchModel)
@@ -493,7 +494,7 @@ class HuyaRegisterRunner:
item = self.batch.items[index]
if not item.db_id:
return
db = SessionLocal()
db = WorkerSessionLocal()
try:
row = (
db.query(HuyaRegisterItemModel)
@@ -536,7 +537,7 @@ class HuyaRegisterRunner:
self, index: int, result: HuyaAutoRegisterResult
) -> tuple[int | None, str, str]:
"""成功时:写账号 + 成功流水(成功一个写一条,立即可导出)。"""
db = SessionLocal()
db = WorkerSessionLocal()
try:
# upsert_huya_cookie 内部会 commit 一次
account = upsert_huya_cookie(
@@ -737,22 +738,30 @@ class HuyaRegisterRunner:
self.batch.message = f"代理白名单预热失败: {msg}"
self._persist_batch_meta()
with ThreadPoolExecutor(max_workers=self.batch.concurrency) as executor:
futures = []
for index in indices:
if self._stop.is_set():
self._set_item(
index,
status="stopped",
message="已停止",
finished_at=_now(),
)
continue
item = self.sms_lines[index]
futures.append(executor.submit(self._run_one, index, item))
futures = []
for index in indices:
if self._stop.is_set():
self._set_item(
index,
status="stopped",
message="已停止",
finished_at=_now(),
)
continue
item = self.sms_lines[index]
future = submit(
TaskLane.HUYA_REGISTER,
self._run_one,
index,
item,
stopped=self._stop.is_set,
)
if future is None:
break
futures.append(future)
for future in as_completed(futures):
future.result()
for future in as_completed(futures):
future.result()
except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
with self._lock:
self.batch.status = "error"
@@ -811,7 +820,7 @@ class HuyaRegisterRegistry:
for index, item in enumerate(sms_lines)
]
db = SessionLocal()
db = WorkerSessionLocal()
try:
batch_row = HuyaRegisterBatchModel(
batch_id=batch_id,
@@ -871,7 +880,7 @@ class HuyaRegisterRegistry:
started_at=_now(),
)
# DB 同步为 running,避免返回 pending 导致前端误判
db = SessionLocal()
db = WorkerSessionLocal()
try:
row = (
db.query(HuyaRegisterBatchModel)
@@ -936,7 +945,7 @@ class HuyaRegisterRegistry:
empty_msg = "没有可继续的未完成/失败条目"
run_label = "续跑未完成"
db = SessionLocal()
db = WorkerSessionLocal()
try:
batch_row = (
db.query(HuyaRegisterBatchModel)
+26 -23
View File
@@ -2,14 +2,14 @@
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import as_completed
from datetime import UTC, datetime
from sqlalchemy.orm import joinedload
from core.huya.cookie_utils import normalize_huya_cookie
from ..database import SessionLocal
from ..database import WorkerSessionLocal
from ..models import HuyaAccount, HuyaTask
from .huya_runner_bind import BindMixin
from .huya_runner_core import ( # noqa: F401 (huya_batch_registry 供 routers 重导出)
@@ -20,6 +20,7 @@ from .huya_runner_elite import EliteMixin
from .huya_runner_goods import GoodsMixin
from .huya_runner_recharge import RechargeMixin
from .huya_service import HUYA_CONFIG_FIELDS, ensure_huya_config, huya_config_value
from .task_runtime import TaskLane, submit
class HuyaBatchRunner(
@@ -34,7 +35,7 @@ class HuyaBatchRunner(
def _execute_one(
self, task_id: int, account_info: dict, config_info: dict, total: int
):
worker_db = SessionLocal()
worker_db = WorkerSessionLocal()
try:
task = worker_db.query(HuyaTask).filter(HuyaTask.id == task_id).first()
account = (
@@ -193,27 +194,29 @@ class HuyaBatchRunner(
self._push_log("result", "")
return
with ThreadPoolExecutor(max_workers=self.concurrency) as executor:
futures = []
for item in task_infos:
if self._stop.is_set():
self._push_log("warning", "任务已停止,跳过剩余账号")
break
futures.append(
executor.submit(
self._execute_one,
item["task_id"],
item["account_info"],
config_info,
total,
)
)
futures = []
for item in task_infos:
if self._stop.is_set():
self._push_log("warning", "任务已停止,跳过剩余账号")
break
future = submit(
TaskLane.HUYA_BATCH,
self._execute_one,
item["task_id"],
item["account_info"],
config_info,
total,
stopped=self._stop.is_set,
)
if future is None:
break
futures.append(future)
for future in as_completed(futures):
try:
future.result()
except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
self._push_log("error", f"虎牙 Worker 异常: {exc}")
for future in as_completed(futures):
try:
future.result()
except Exception as exc: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
self._push_log("error", f"虎牙 Worker 异常: {exc}")
self._push_log("info", f"虎牙批次 {self.batch_id} 完成")
self._push_log("result", "")
+2 -2
View File
@@ -90,10 +90,10 @@ class HuyaBatchRunnerCore:
account_id = getattr(self._wss_sessions, "account_id", None)
if not account_id:
return
from ..database import SessionLocal
from ..database import WorkerSessionLocal
from .huya_service import persist_huya_session_cookie
db = SessionLocal()
db = WorkerSessionLocal()
try:
if persist_huya_session_cookie(
db, int(account_id), original_cookie, filled_cookie
+28 -17
View File
@@ -14,9 +14,10 @@ from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from loguru import logger
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from ...database import SessionLocal
from ...database import WorkerSessionLocal
from ...models import IntegrationOutbox
from .security import build_webhook_signature
@@ -74,19 +75,29 @@ def enqueue_integration_event(
if existing is not None:
return False
db.add(
IntegrationOutbox(
event_id=event_id,
destination=destination,
event_type=event_type,
dedupe_key=normalized_dedupe,
payload_json=body,
status="pending",
attempts=0,
next_attempt_at=datetime.now(UTC),
created_at=datetime.now(UTC),
)
)
try:
with db.begin_nested():
db.add(
IntegrationOutbox(
event_id=event_id,
destination=destination,
event_type=event_type,
dedupe_key=normalized_dedupe,
payload_json=body,
status="pending",
attempts=0,
next_attempt_at=datetime.now(UTC),
created_at=datetime.now(UTC),
)
)
db.flush()
except IntegrityError:
# A concurrent transaction may have inserted the same stable dedupe key
# after our pre-check. The nested transaction keeps the outer business
# transaction usable and treats the event as already enqueued.
if normalized_dedupe:
return False
raise
return True
@@ -118,7 +129,7 @@ def _env_int(name: str, default: int, minimum: int, maximum: int) -> int:
def _claim_due_events(limit: int) -> list[_ClaimedEvent]:
db = SessionLocal()
db = WorkerSessionLocal()
now = datetime.now(UTC)
try:
(
@@ -202,7 +213,7 @@ def _deliver(event: _ClaimedEvent, base_url: str, integration_key: str) -> None:
def _mark_sent(event_id: str) -> None:
db = SessionLocal()
db = WorkerSessionLocal()
try:
(
db.query(IntegrationOutbox)
@@ -229,7 +240,7 @@ def _mark_failed(event: _ClaimedEvent, error: Exception) -> None:
)
dead = attempts >= max_attempts
message = str(error)[:2000]
db = SessionLocal()
db = WorkerSessionLocal()
try:
(
db.query(IntegrationOutbox)
+34 -33
View File
@@ -5,7 +5,7 @@ import os
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import as_completed
from datetime import UTC, datetime
from types import SimpleNamespace
from typing import cast
@@ -32,6 +32,7 @@ from .douyu_account_probe import persist_probe_result, probe_session
from .douyu_service import LOGIN_ATTEMPT_IN_FLIGHT_STATUSES
from .login_failure import classify_failure, failure_kind_label, is_account_fault
from .proxy_service import build_proxy_runtime
from .task_runtime import TaskLane, submit
CHECK_STATUS_MESSAGES = {
"account_cancelled": "账号已注销",
@@ -296,7 +297,7 @@ def _persist_login_cookie_verification(
class LoginBatchRunner:
"""批量登录执行器,在线程中运行,通过 ThreadPoolExecutor 并发登录多个账号。"""
"""批量登录执行器,通过进程级登录槽并发执行账号。"""
def __init__(
self,
@@ -357,7 +358,7 @@ class LoginBatchRunner:
message: str = "重新登录已停止,旧 Cookie 已保留",
) -> None:
"""把未开始或已中断的重登尝试收敛为终态,旧 CK 原封不动。"""
worker_db = SessionLocal()
worker_db = WorkerSessionLocal()
try:
task = worker_db.query(LoginTask).filter(LoginTask.id == task_id).first()
if task and _attempt_in_flight(task):
@@ -376,7 +377,7 @@ class LoginBatchRunner:
与 `_mark_relogin_stopped` 的区别只是入口:批次挂了但记录还没建出来时,
这里是唯一能把「该账号的在途尝试」收干净的路径。
"""
worker_db = SessionLocal()
worker_db = WorkerSessionLocal()
try:
tasks = (
worker_db.query(LoginTask)
@@ -432,7 +433,7 @@ class LoginBatchRunner:
self._mark_relogin_stopped(task_id, "任务已停止,旧 Cookie 已保留")
return
worker_db = SessionLocal()
worker_db = WorkerSessionLocal()
try:
task = worker_db.query(LoginTask).filter(LoginTask.id == task_id).first()
if not task:
@@ -774,33 +775,33 @@ class LoginBatchRunner:
self._push_log("result", "")
return
# 并发执行登录,每个账号独立获取代理
with ThreadPoolExecutor(max_workers=concurrency) as executor:
futures = []
for item_index, item in enumerate(task_infos):
if self._stop.is_set():
self._push_log("warning", "任务已停止,跳过剩余账号")
# 所有模式都要把没轮到的尝试收敛掉,否则会一直停在
# 「等待中」;已有 CK 的记录本身不受影响。
for pending_item in task_infos[item_index:]:
self._mark_relogin_stopped(
pending_item["task_id"], "任务已停止,旧 Cookie 已保留"
)
break
future = executor.submit(
self._execute_one,
item["task_id"],
item["acc_info"],
total,
)
futures.append(future)
# 全局登录槽负责并发;批次只提交任务并收集结果。
futures = []
for item_index, item in enumerate(task_infos):
if self._stop.is_set():
self._push_log("warning", "任务已停止,跳过剩余账号")
for pending_item in task_infos[item_index:]:
self._mark_relogin_stopped(
pending_item["task_id"], "任务已停止,旧 Cookie 已保留"
)
break
future = submit(
TaskLane.LOGIN,
self._execute_one,
item["task_id"],
item["acc_info"],
total,
stopped=self._stop.is_set,
)
if future is None:
break
futures.append(future)
# 等待所有任务完成
for future in as_completed(futures):
try:
future.result()
except Exception as e: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
self._push_log("error", f"Worker 异常: {e}")
for future in as_completed(futures):
try:
future.result()
except Exception as e: # noqa: BLE001 外部接口与任务边界需要保留宽泛异常兜底
self._push_log("error", f"Worker 异常: {e}")
self._push_log("info", f"批量{action_name}任务 {batch_id} 完成")
self._push_log("result", "")
@@ -902,5 +903,5 @@ def cleanup_orphan_login_tasks(
return _converge_interrupted_attempts(db, message)
# 在模块末尾导入 SessionLocal(避免循环导入)
from ..database import SessionLocal
# 在模块末尾导入 WorkerSessionLocal(避免循环导入)
from ..database import WorkerSessionLocal
@@ -2,6 +2,7 @@
from __future__ import annotations
import asyncio
import os
import threading
from collections.abc import Iterable
@@ -12,7 +13,7 @@ from loguru import logger
from sqlalchemy import or_
from sqlalchemy.orm import Session
from ...database import SessionLocal
from ...database import WorkerSessionLocal
from ...models import PlatformFulfillmentTask
from . import service
@@ -142,6 +143,21 @@ def run_auto_allocation_once(
if allocated_task is not None:
allocated_task.next_allocate_at = None
db.commit()
try:
asyncio.run(
service.start_binding(
db,
task_id=candidate_task_id,
current=actor,
)
)
except Exception as exc: # noqa: BLE001 - allocation remains valid
db.rollback()
logger.warning(
"直播账号履约自动开始绑定失败 task_id={}: {}",
candidate_task_id,
exc,
)
else:
failed += 1
_schedule_retry(
@@ -171,6 +187,48 @@ def run_auto_allocation_once(
}
def start_allocated_bindings_once(
db: Session,
*,
limit: int = DEFAULT_BATCH_SIZE,
) -> int:
"""Start durable binding preparation for allocated, unstarted sessions."""
rows = (
db.query(PlatformFulfillmentTask)
.filter(
PlatformFulfillmentTask.execution_status == "allocated",
PlatformFulfillmentTask.binding_status == "not_started",
PlatformFulfillmentTask.account_id.isnot(None),
PlatformFulfillmentTask.lease_id.isnot(None),
)
.order_by(PlatformFulfillmentTask.updated_at.asc(), PlatformFulfillmentTask.id.asc())
.limit(max(1, limit))
.all()
)
if not rows:
return 0
actor = service._integration_actor(db)
started = 0
for task in rows:
try:
asyncio.run(
service.start_binding(
db,
task_id=int(task.id),
current=actor,
)
)
started += 1
except Exception as exc: # noqa: BLE001 - one session cannot block the rest
db.rollback()
logger.warning(
"直播账号履约补启动绑定失败 task_id={}: {}",
task.id,
exc,
)
return started
def _run() -> None:
interval = _env_int(
"PLATFORM_FULFILLMENT_AUTO_ALLOCATE_SECONDS",
@@ -179,7 +237,7 @@ def _run() -> None:
3600,
)
while not _stop_event.is_set():
db = SessionLocal()
db = WorkerSessionLocal()
try:
pending_ids = _take_pending_task_ids(
_env_int(
@@ -190,8 +248,11 @@ def _run() -> None:
)
)
result = run_auto_allocation_once(db, task_ids=pending_ids or None)
started = start_allocated_bindings_once(db)
if result["checked"]:
logger.info("直播账号履约自动选号: {}", result)
if started:
logger.info("直播账号履约补启动绑定: {} 条", started)
except Exception as exc: # noqa: BLE001 - worker 不能因单轮异常退出
db.rollback()
logger.warning("直播账号履约自动选号轮次失败: {}", exc)
@@ -29,6 +29,8 @@ def _state_dedupe_key(payload: dict[str, Any]) -> str:
"platName": payload.get("platName"),
"bindReadyForConfirm": payload.get("bindReadyForConfirm"),
"bindConfirmed": payload.get("bindConfirmed"),
"queueState": payload.get("queueState"),
"queueLane": payload.get("queueLane"),
"customerProfileReset": payload.get("customerProfileReset"),
}
digest = hashlib.sha256(
@@ -7,7 +7,7 @@ import threading
from loguru import logger
from ...database import SessionLocal
from ...database import WorkerSessionLocal
_stop_event = threading.Event()
_thread: threading.Thread | None = None
@@ -31,7 +31,7 @@ def _run() -> None:
# 不抢占启动时的 Webhook;首轮延后一分钟。
if _stop_event.wait(min(60, _interval_seconds())):
break
db = SessionLocal()
db = WorkerSessionLocal()
try:
queued = 0
for _page in range(20):
@@ -12,6 +12,7 @@ from typing import Any
from fastapi import HTTPException
from loguru import logger
from sqlalchemy import or_
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from ...models import (
@@ -763,6 +764,19 @@ def _load_task(db: Session, task_id: int) -> PlatformFulfillmentTask:
return task
def _load_task_for_update(db: Session, task_id: int) -> PlatformFulfillmentTask:
task = (
db.query(PlatformFulfillmentTask)
.filter(PlatformFulfillmentTask.id == int(task_id))
.populate_existing()
.with_for_update()
.first()
)
if task is None:
raise HTTPException(status_code=404, detail="直播账号履约任务不存在")
return task
def _retain_lease_for_active_collab(
db: Session,
*,
@@ -1241,6 +1255,40 @@ def _binding_failure_issue(bind_task: DouyuTask, result: dict[str, Any]) -> str:
return task_issue if messages.get(task_issue) else "binding_failed"
def _customer_binding_queue_state(
task: PlatformFulfillmentTask,
bind_task: DouyuTask | None,
*,
lease_ready: bool,
) -> tuple[str, int]:
if task.execution_status == "completed":
return "completed", 0
if task.execution_status == "cancelled":
return "cancelled", 0
if task.issue_code == "mapping_missing":
return "mapping_missing", 0
if not task.account_id or not task.lease_id:
return "allocation_queued", 2000
if not lease_ready:
return "lease_inactive", 0
if bind_task is None:
return "ready_to_start", 0
result = bind_task.result or {}
if bool(result.get("bind_confirmed")) or task.binding_status == "bound":
return "confirmed", 0
if pending_binding_role_from_result(result):
return "role_ready", 0
if bind_task.status in {"failed", "error", "stopped"}:
return "failed", 0
if task.binding_status == "confirming":
return "confirming", 1000
if bind_task.status in {"planned", "pending"}:
return "qr_queued", 1000
if result.get("url"):
return "waiting_scan", 5000 if result.get("bind_polling") else 0
return "preparing", 1000
def customer_binding_out(db: Session, task_id: int) -> dict[str, Any]:
"""面向前端客户页的安全绑定快照,不包含账号、CK、积分和租约。"""
task = _load_task(db, task_id)
@@ -1259,6 +1307,11 @@ def customer_binding_out(db: Session, task_id: int) -> dict[str, Any]:
lease_ready = bool(task.account_id and lease and lease.status == "active")
binding_failed = task.binding_status == "failed"
binding_confirmed = task.binding_status == "bound"
queue_state, retry_after_ms = _customer_binding_queue_state(
task,
bind_task,
lease_ready=lease_ready,
)
return {
"sessionId": task.id,
"platformOrderId": task.platform_order_id,
@@ -1282,6 +1335,10 @@ def customer_binding_out(db: Session, task_id: int) -> dict[str, Any]:
"platName": str(bind_result.get("plat_name") or ""),
"bindReadyForConfirm": (bool(pending_role) and not binding_failed),
"bindConfirmed": bool(bind_result.get("bind_confirmed")) and binding_confirmed,
"queueState": queue_state,
"retryAfterMs": retry_after_ms,
"queueLane": bind_task.queue_lane if bind_task else "",
"accepted": False,
"stateVersion": int(task.state_version or 0),
"completedAt": _jsonable(task.completed_at),
"updatedAt": _jsonable(task.updated_at),
@@ -1318,8 +1375,9 @@ def ensure_customer_session(
if task.execution_status in {"pending_mapping", "ready", "manual_required"}:
if task.issue_code == "mapping_missing":
return customer_binding_out(db, task.id)
actor = _integration_actor(db)
allocate_task(db, task_id=task.id, current=actor)
from .auto_allocator import wake_auto_allocator
wake_auto_allocator([int(task.id)])
return customer_binding_out(db, task.id)
@@ -1330,18 +1388,40 @@ async def customer_start_binding(
force_refresh: bool = False,
) -> dict[str, Any]:
task = _load_task(db, task_id)
if task.execution_status in {"completed", "cancelled"}:
return customer_binding_out(db, task.id)
if task.issue_code == "mapping_missing":
return customer_binding_out(db, task.id)
if not task.account_id or not task.lease_id:
from .auto_allocator import wake_auto_allocator
wake_auto_allocator([int(task.id)])
payload = customer_binding_out(db, task.id)
payload["accepted"] = True
payload["queueState"] = "allocation_queued"
payload["retryAfterMs"] = 2000
return payload
lease = db.query(AccountLease).filter(AccountLease.id == task.lease_id).first()
if lease is None or lease.status != "active":
return customer_binding_out(db, task.id)
previous_bind_task_id = task.bind_task_id
actor = _integration_actor(db)
if task.execution_status not in {"completed", "cancelled"} and (
not task.account_id or not task.lease_id
):
allocate_task(db, task_id=task.id, current=actor)
await start_binding(
db,
task_id=task.id,
current=actor,
force_refresh=force_refresh,
)
return customer_binding_out(db, task.id)
payload = customer_binding_out(db, task.id)
accepted = task.bind_task_id != previous_bind_task_id
payload["accepted"] = accepted
if accepted:
payload["queueState"] = "qr_queued"
payload["retryAfterMs"] = 1000
return payload
def customer_refresh_binding(db: Session, *, task_id: int) -> dict[str, Any]:
@@ -1881,7 +1961,7 @@ async def start_binding(
current: User,
force_refresh: bool = False,
) -> dict[str, Any]:
task = _load_task(db, task_id)
task = _load_task_for_update(db, task_id)
if not task.account_id or not task.lease_id:
raise HTTPException(status_code=409, detail="请先分配账号")
lease = db.query(AccountLease).filter(AccountLease.id == task.lease_id).first()
@@ -1924,9 +2004,16 @@ async def start_binding(
message="等待生成绑定二维码",
result={"payload": {}},
created_by=current.id,
platform_task_id=task.id,
)
db.add(bind_task)
db.flush()
try:
db.flush()
except IntegrityError:
# The database uniqueness gate wins a race that started before either
# request could acquire the fulfillment row lock.
db.rollback()
return get_task_detail(db, task_id)
account = db.query(Account).filter(Account.id == task.account_id).first()
if account is not None:
record_douyu_task_started(
@@ -2113,7 +2200,7 @@ def reconcile_binding_state_outbox(
async def confirm_binding(
db: Session, *, task_id: int, current: User
) -> dict[str, Any]:
task = _load_task(db, task_id)
task = _load_task_for_update(db, task_id)
task = _load_task(db, refresh_binding(db, task_id=task.id)["task"]["id"])
if task.binding_status != "role_ready":
raise HTTPException(status_code=409, detail="当前没有待确认的游戏角色")
@@ -2146,6 +2233,7 @@ async def confirm_binding(
"baseline_role_name": baseline_role_name,
},
created_by=current.id,
platform_task_id=task.id,
)
db.add(confirm_task)
db.flush()
+195
View File
@@ -0,0 +1,195 @@
"""Bounded, process-local execution lanes for background work.
Each lane owns its workers and admission budget. A slow or saturated domain can
therefore no longer consume every worker slot in the process. This runtime is
intended for the current single-Uvicorn deployment; durable scheduling remains
in the database.
"""
from __future__ import annotations
import os
import threading
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
from enum import StrEnum
from typing import Any
class TaskLane(StrEnum):
INTERACTIVE_BINDING = "interactive_binding"
BINDING_POLL = "binding_poll"
LOGIN = "login"
ACCOUNT_CHECK = "account_check"
AUTOMATION = "automation"
QUERY_BATCH = "query_batch"
COLLAB_RECOMMEND = "collab_recommend"
PAYMENT_WAIT = "payment_wait"
OPS_WRITE = "ops_write"
HUYA_BATCH = "huya_batch"
HUYA_REGISTER = "huya_register"
INTEGRATION = "integration"
MAINTENANCE = "maintenance"
def _env_int(name: str, default: int, minimum: int = 1) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
return default
return max(minimum, value)
@dataclass(frozen=True)
class LaneConfig:
workers: int
max_in_flight: int
_DEFAULT_WORKERS: dict[TaskLane, int] = {
TaskLane.INTERACTIVE_BINDING: 3,
TaskLane.BINDING_POLL: 2,
TaskLane.LOGIN: 4,
TaskLane.ACCOUNT_CHECK: 4,
TaskLane.AUTOMATION: 2,
TaskLane.QUERY_BATCH: 3,
TaskLane.COLLAB_RECOMMEND: 2,
TaskLane.PAYMENT_WAIT: 2,
TaskLane.OPS_WRITE: 2,
TaskLane.HUYA_BATCH: 2,
TaskLane.HUYA_REGISTER: 2,
TaskLane.INTEGRATION: 2,
TaskLane.MAINTENANCE: 1,
}
def _lane_config(lane: TaskLane) -> LaneConfig:
key = lane.value.upper()
workers = _env_int(f"TASK_LANE_{key}_WORKERS", _DEFAULT_WORKERS[lane])
max_in_flight = _env_int(
f"TASK_LANE_{key}_MAX_IN_FLIGHT",
max(workers * 32, workers),
)
return LaneConfig(workers=workers, max_in_flight=max(max_in_flight, workers))
@dataclass
class _LaneRuntime:
config: LaneConfig
executor: ThreadPoolExecutor
admission: threading.BoundedSemaphore
_LOCK = threading.RLock()
_RUNTIMES: dict[TaskLane, _LaneRuntime] = {}
def _runtime(lane: TaskLane) -> _LaneRuntime:
with _LOCK:
runtime = _RUNTIMES.get(lane)
if runtime is not None:
return runtime
config = _lane_config(lane)
runtime = _LaneRuntime(
config=config,
executor=ThreadPoolExecutor(
max_workers=config.workers,
thread_name_prefix=f"lane-{lane.value}",
),
admission=threading.BoundedSemaphore(config.max_in_flight),
)
_RUNTIMES[lane] = runtime
return runtime
def submit[T](
lane: TaskLane,
fn: Callable[..., T],
/,
*args: Any,
stopped: Callable[[], bool] | None = None,
**kwargs: Any,
) -> Future[T] | None:
"""Submit work to a lane, waiting for admission when the lane is full.
Batch coordinators are cheap threads and are allowed to block here. Worker
threads themselves must not recursively submit to the same lane.
"""
runtime = _runtime(lane)
while True:
if stopped is not None and stopped():
return None
if runtime.admission.acquire(timeout=0.2):
break
def run() -> T:
try:
return fn(*args, **kwargs)
finally:
runtime.admission.release()
return runtime.executor.submit(run)
def lane_stats() -> dict[str, dict[str, int]]:
with _LOCK:
runtimes = list(_RUNTIMES.items())
stats: dict[str, dict[str, int]] = {}
for lane, runtime in runtimes:
stats[lane.value] = {
"workers": runtime.config.workers,
"max_in_flight": runtime.config.max_in_flight,
"admission_value": runtime.admission._value,
}
return stats
def shutdown(wait: bool = False) -> None:
with _LOCK:
runtimes = list(_RUNTIMES.values())
_RUNTIMES.clear()
for runtime in runtimes:
runtime.executor.shutdown(wait=wait, cancel_futures=False)
def douyu_lane_for_task(task_type: str) -> TaskLane:
if task_type in {
"get_bind_qr",
"confirm_bind",
"prepare_esports_bind",
"get_esports_bind_qr",
"confirm_esports_bind",
"get_xpd_bind_qr",
"query_xpd_bind_info",
"confirm_xpd_bind",
"query_xpd_role",
}:
return TaskLane.INTERACTIVE_BINDING
if task_type in {
"create_elite_qr",
"create_esports_qr",
"create_gold_qr",
}:
return TaskLane.PAYMENT_WAIT
if task_type in {
"lock_goods",
"pay_locked_order",
"exchange_goods",
"exchange_esports_goods",
"exchange_xpd_goods",
"donate_elite_gift",
"donate_esports_chicken_gift",
"donate_esports_firework_gift",
}:
return TaskLane.OPS_WRITE
return TaskLane.QUERY_BATCH
__all__ = [
"TaskLane",
"douyu_lane_for_task",
"lane_stats",
"shutdown",
"submit",
]