Files
live-hub-py/web/backend/services/account_work_queue.py
T

294 lines
7.9 KiB
Python

"""账号内部维护工作队列。"""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from ..models import Account, AccountWorkItem
def active_account_work_exists(*, work_type: str):
"""SQL 子查询:同一账号已有同类型的待执行或运行中工作。"""
return (
select(AccountWorkItem.id)
.where(
AccountWorkItem.account_id == Account.id,
AccountWorkItem.work_type == work_type,
AccountWorkItem.status.in_(("pending", "running")),
)
.exists()
)
def active_account_work_count(
db: Session,
*,
work_types: tuple[str, ...],
) -> int:
"""当前账号同步队列中尚未结束的工作数。"""
if not work_types:
return 0
return int(
db.query(func.count(AccountWorkItem.id))
.filter(
AccountWorkItem.work_type.in_(work_types),
AccountWorkItem.status.in_(("pending", "running")),
)
.scalar()
or 0
)
def enqueue_account_work(
db: Session,
*,
account_id: int,
work_type: str,
priority: int = 100,
due_at: datetime | None = None,
max_attempts: int = 3,
reason: str = "",
payload: dict[str, Any] | None = None,
) -> AccountWorkItem:
dedupe_key = f"{work_type}:{account_id}"
existing = (
db.query(AccountWorkItem)
.filter(
AccountWorkItem.account_id == account_id,
AccountWorkItem.work_type == work_type,
AccountWorkItem.status.in_(("pending", "running")),
)
.first()
)
if existing is not None:
existing.priority = min(existing.priority, priority)
existing.max_attempts = max(existing.max_attempts, max(1, int(max_attempts)))
if due_at is not None and due_at < existing.due_at:
existing.due_at = due_at
db.flush()
return existing
item = AccountWorkItem(
account_id=account_id,
work_type=work_type[:32],
priority=int(priority),
# MySQL DATETIME 不保存微秒;立即到期的工作项必须落在当前秒,
# 否则四舍五入到下一秒后,本轮 claim 会误判“还没到期”。
due_at=(due_at or datetime.now(UTC)).replace(microsecond=0),
max_attempts=max(1, int(max_attempts)),
dedupe_key=dedupe_key[:191],
reason=reason[:255],
payload_json=payload,
)
try:
with db.begin_nested():
db.add(item)
db.flush()
return item
except IntegrityError:
return (
db.query(AccountWorkItem)
.filter(AccountWorkItem.active_dedupe_key == dedupe_key[:191])
.one()
)
def claim_account_work(
db: Session,
*,
work_types: tuple[str, ...],
limit: int = 10,
now: datetime | None = None,
account_ids: tuple[int, ...] | None = None,
) -> list[AccountWorkItem]:
moment = now or datetime.now(UTC)
query = (
db.query(AccountWorkItem)
.filter(
AccountWorkItem.work_type.in_(work_types),
AccountWorkItem.status == "pending",
AccountWorkItem.due_at <= moment,
)
)
if account_ids is not None:
if not account_ids:
return []
query = query.filter(AccountWorkItem.account_id.in_(account_ids))
rows = (
query
.order_by(
AccountWorkItem.priority.asc(),
AccountWorkItem.due_at.asc(),
AccountWorkItem.id.asc(),
)
.with_for_update(skip_locked=True)
.limit(max(1, int(limit)))
.all()
)
for row in rows:
row.status = "running"
row.started_at = moment
row.attempt_count = int(row.attempt_count or 0) + 1
row.updated_at = moment
if rows:
db.flush()
return rows
def complete_account_work(
db: Session,
item: AccountWorkItem,
*,
now: datetime | None = None,
) -> None:
moment = now or datetime.now(UTC)
item.status = "success"
item.finished_at = moment
item.last_error = ""
item.updated_at = moment
db.flush()
def complete_account_work_type(
db: Session,
*,
account_id: int,
work_type: str,
now: datetime | None = None,
) -> int:
"""完成某个账号指定类型的待办;成功后旧待办不再阻塞后续工作。"""
moment = now or datetime.now(UTC)
items = (
db.query(AccountWorkItem)
.filter(
AccountWorkItem.account_id == account_id,
AccountWorkItem.work_type == work_type,
AccountWorkItem.status.in_(("pending", "running")),
)
.all()
)
for item in items:
complete_account_work(db, item, now=moment)
return len(items)
def fail_account_work(
db: Session,
item: AccountWorkItem,
*,
error: BaseException | str,
retry_after_seconds: int = 300,
now: datetime | None = None,
) -> None:
moment = now or datetime.now(UTC)
message = str(error)[:2000]
item.last_error = message
item.updated_at = moment
if int(item.attempt_count or 0) >= int(item.max_attempts or 1):
item.status = "failed"
item.finished_at = moment
else:
item.status = "pending"
item.due_at = (
moment + timedelta(seconds=max(1, retry_after_seconds))
).replace(microsecond=0)
db.flush()
def recover_running_account_work(
db: Session,
*,
now: datetime | None = None,
) -> int:
"""服务重启后把中断的 running 工作项退回 pending。"""
moment = (now or datetime.now(UTC)).replace(microsecond=0)
items = (
db.query(AccountWorkItem)
.filter(AccountWorkItem.status == "running")
.all()
)
for item in items:
item.status = "pending"
item.due_at = moment
item.last_error = "工作执行中断(服务重启),已重新排队"
item.updated_at = moment
if items:
db.commit()
return len(items)
def reschedule_account_work_type(
db: Session,
*,
account_id: int,
work_type: str,
due_at: datetime,
reason: str = "",
) -> int:
"""把自动重登失败的工作项按账号退避时间重新排队。"""
moment = (due_at if due_at.tzinfo else due_at.replace(tzinfo=UTC)).replace(
microsecond=0
)
items = (
db.query(AccountWorkItem)
.filter(
AccountWorkItem.account_id == account_id,
AccountWorkItem.work_type == work_type,
AccountWorkItem.status.in_(("pending", "running")),
)
.all()
)
for item in items:
item.status = "pending"
item.due_at = moment
item.last_error = reason[:2000]
item.updated_at = datetime.now(UTC)
db.flush()
return len(items)
def fail_account_work_type(
db: Session,
*,
account_id: int,
work_type: str,
reason: str,
now: datetime | None = None,
) -> int:
"""关闭不再自动处理的工作项,例如重登已转人工。"""
moment = now or datetime.now(UTC)
items = (
db.query(AccountWorkItem)
.filter(
AccountWorkItem.account_id == account_id,
AccountWorkItem.work_type == work_type,
AccountWorkItem.status.in_(("pending", "running")),
)
.all()
)
for item in items:
item.status = "failed"
item.finished_at = moment
item.last_error = reason[:2000]
item.updated_at = moment
db.flush()
return len(items)
__all__ = [
"active_account_work_count",
"active_account_work_exists",
"claim_account_work",
"complete_account_work",
"complete_account_work_type",
"enqueue_account_work",
"fail_account_work",
"fail_account_work_type",
"recover_running_account_work",
"reschedule_account_work_type",
]