perf(douyu): 优化慢查询并修复供应商充值收口
This commit is contained in:
@@ -114,6 +114,7 @@ def register_default_jobs() -> None:
|
||||
batch_size=2000,
|
||||
concurrency=1,
|
||||
run=run_douyu_tasks_retention,
|
||||
default_enabled=True,
|
||||
)
|
||||
)
|
||||
register_job(
|
||||
|
||||
@@ -84,14 +84,6 @@ def build_select_query(
|
||||
)
|
||||
|
||||
|
||||
def count_remaining(
|
||||
db: Session,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> int:
|
||||
return _base_query(db, now or datetime.now(UTC)).count()
|
||||
|
||||
|
||||
def run_account_capability_refresh(
|
||||
db: Session,
|
||||
job: AutomationJob,
|
||||
@@ -103,7 +95,9 @@ def run_account_capability_refresh(
|
||||
job,
|
||||
accounts,
|
||||
now=now,
|
||||
remaining_count=count_remaining(db, now=now),
|
||||
# 精确剩余量要完整评估 accounts × commodity EXISTS,生产实测单次约
|
||||
# 1.3 秒、扫描 38 万行;它只用于展示,直接省略,避免每轮制造慢查询。
|
||||
remaining_count=None,
|
||||
source="capability_demand_refresh",
|
||||
log_label="capability",
|
||||
empty_message="当前履约商品没有到期待刷新的账号能力",
|
||||
@@ -113,6 +107,5 @@ def run_account_capability_refresh(
|
||||
__all__ = [
|
||||
"JOB_KEY",
|
||||
"build_select_query",
|
||||
"count_remaining",
|
||||
"run_account_capability_refresh",
|
||||
]
|
||||
|
||||
@@ -34,7 +34,8 @@
|
||||
- 不删行、只删「过期的整行」:不做「把 result 清空保留行」这种半吊子操作,
|
||||
因为部分前端派生**明确要求 result 非空**,清空等于制造「有行但读不出状态」的
|
||||
更隐蔽的坏状态;
|
||||
- 作业**默认关闭**(`ensure_jobs` 建行时 enabled=False),由运维在页面上打开。
|
||||
- 作业**默认启用**(`ensure_jobs` 建行时 enabled=True),避免执行日志无限增长;
|
||||
运维仍可在页面上关闭。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
自动化按「渐进上线」推进,因此:
|
||||
|
||||
- 调度器总开关 ``AUTOMATION_ENABLED`` **默认关闭**;
|
||||
- 每个作业自己还有开关,**默认关闭**,在 ``/automation`` 页面打开;
|
||||
- 每个作业自己还有开关,**默认关闭**;少数维护型作业可通过
|
||||
``JobSpec.default_enabled`` 在首次建行时默认启用,之后仍完全尊重页面配置;
|
||||
- 作业状态全部落库(``automation_jobs`` / ``automation_runs``),进程重启不丢
|
||||
调度进度,也不会因为重启而重复跑一轮。
|
||||
|
||||
@@ -65,6 +66,8 @@ class JobSpec:
|
||||
#: **多久跑一次**(调度频率)。默认 5 分钟,与改动前的实际行为一致。
|
||||
#: ⚠️ 必须放在**有默认值**的位置(dataclass 不允许默认值字段排在无默认值之前)。
|
||||
run_interval_seconds: int = 300
|
||||
#: 新库首次创建作业行时是否默认启用。已有行永远尊重运维配置。
|
||||
default_enabled: bool = False
|
||||
|
||||
|
||||
_REGISTRY: dict[str, JobSpec] = {}
|
||||
@@ -120,7 +123,7 @@ def ensure_jobs(db: Session) -> None:
|
||||
AutomationJob(
|
||||
key=spec.key,
|
||||
title=spec.title,
|
||||
enabled=False,
|
||||
enabled=spec.default_enabled,
|
||||
interval_seconds=spec.interval_seconds,
|
||||
run_interval_seconds=spec.run_interval_seconds,
|
||||
batch_size=spec.batch_size,
|
||||
|
||||
@@ -63,7 +63,22 @@ class DouyuBatchRunner(
|
||||
)
|
||||
return
|
||||
if task.task_type == "create_gold_qr":
|
||||
self._poll_create_gold_qr_once(db, task, account, cookie, config)
|
||||
if str(result.get("recharge_channel") or "") == "supplier_api":
|
||||
amount = int(
|
||||
result.get("buy_num")
|
||||
or self._task_payload(task).get("amount")
|
||||
or self._task_payload(task).get("gold_amount")
|
||||
or 1
|
||||
)
|
||||
self._poll_create_gold_supplier_once(
|
||||
db,
|
||||
task,
|
||||
account,
|
||||
cookie,
|
||||
amount=amount,
|
||||
)
|
||||
else:
|
||||
self._poll_create_gold_qr_once(db, task, account, cookie, config)
|
||||
elif task.task_type == "create_elite_qr":
|
||||
self._poll_create_elite_qr_once(db, task, account, cookie, config)
|
||||
elif task.task_type == "create_esports_qr":
|
||||
|
||||
@@ -197,9 +197,11 @@ class GoldMixin:
|
||||
db: Session,
|
||||
task: DouyuTask,
|
||||
account: Account,
|
||||
cookie: str,
|
||||
*,
|
||||
amount: int,
|
||||
) -> None:
|
||||
"""查询一次供应商直充订单状态。"""
|
||||
"""同时核对供应商订单和斗鱼余额,任一证明到账即可收口。"""
|
||||
db.refresh(task)
|
||||
if task.status in {"success", "failed", "error", "stopped"}:
|
||||
return
|
||||
@@ -208,6 +210,20 @@ class GoldMixin:
|
||||
order_no = str(result["out_order_id"])
|
||||
result["payment_polling"] = True
|
||||
result["payment_poll_count"] = int(result.get("payment_poll_count") or 0) + 1
|
||||
code: int | None = None
|
||||
status: int | None = None
|
||||
supplier_error = ""
|
||||
gold_result: dict[str, Any] = {}
|
||||
gold_error = ""
|
||||
|
||||
try:
|
||||
gold_result = self._refresh_account_gold_balance(
|
||||
self._client(cookie, direct=True),
|
||||
account,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 余额核对失败仍继续查供应商
|
||||
gold_error = str(exc)
|
||||
|
||||
try:
|
||||
client = FishFinRechargeClient(FishFinRechargeConfig.from_env())
|
||||
payload = client.query_order(order_no)
|
||||
@@ -220,62 +236,115 @@ class GoldMixin:
|
||||
"supplier_order": self._supplier_result(payload),
|
||||
}
|
||||
)
|
||||
# 查询期间供应商回调可能已经把任务收口。重新锁行后再决定是否写入,
|
||||
# 避免长时间 HTTP 调用结束后用旧状态覆盖回调结果。
|
||||
db.rollback()
|
||||
locked = (
|
||||
db.query(DouyuTask)
|
||||
.filter(
|
||||
DouyuTask.id == task_id,
|
||||
DouyuTask.status == "running",
|
||||
DouyuTask.queue_lane == "payment_poll",
|
||||
)
|
||||
.with_for_update()
|
||||
.first()
|
||||
except Exception as exc: # noqa: BLE001 供应商查询异常由持久调度继续重试
|
||||
supplier_error = str(exc)
|
||||
|
||||
# 查询期间供应商回调可能已经把任务收口。重新锁行后再决定是否写入,
|
||||
# 避免长时间 HTTP 调用结束后用旧状态覆盖回调结果。
|
||||
db.rollback()
|
||||
locked = (
|
||||
db.query(DouyuTask)
|
||||
.filter(
|
||||
DouyuTask.id == task_id,
|
||||
DouyuTask.status == "running",
|
||||
DouyuTask.queue_lane == "payment_poll",
|
||||
)
|
||||
if locked is None:
|
||||
return
|
||||
task = locked
|
||||
locked_account = (
|
||||
db.query(Account).filter(Account.id == task.account_id).first()
|
||||
.with_for_update()
|
||||
.first()
|
||||
)
|
||||
if locked is None:
|
||||
return
|
||||
task = locked
|
||||
locked_account = db.query(Account).filter(Account.id == task.account_id).first()
|
||||
if locked_account is None:
|
||||
self._mark_task(
|
||||
db,
|
||||
task,
|
||||
"failed",
|
||||
"账号不存在,无法继续核对供应商充值到账",
|
||||
result,
|
||||
)
|
||||
if locked_account is not None:
|
||||
account = locked_account
|
||||
if code != 200:
|
||||
self._finish_supplier_gold_order(
|
||||
db,
|
||||
task,
|
||||
account,
|
||||
result,
|
||||
amount=amount,
|
||||
status=(status if status in {2, 3, 4} else 4),
|
||||
)
|
||||
return
|
||||
if status in {2, 3, 4}:
|
||||
self._finish_supplier_gold_order(
|
||||
db,
|
||||
task,
|
||||
account,
|
||||
result,
|
||||
amount=amount,
|
||||
status=status,
|
||||
)
|
||||
return
|
||||
return
|
||||
account = locked_account
|
||||
if gold_result:
|
||||
result.update(gold_result)
|
||||
account.gold_balance = self._to_int(gold_result.get("gold_balance"))
|
||||
account.exchange_balance = self._to_int(gold_result.get("exchange_balance"))
|
||||
account.updated_at = datetime.now(UTC)
|
||||
|
||||
baseline = self._to_int(result.get("baseline_gold_balance"))
|
||||
baseline_ready = bool(
|
||||
result.get("baseline_gold_ready")
|
||||
and result.get("baseline_gold_source") == "pre_order"
|
||||
and baseline is not None
|
||||
)
|
||||
current_gold = (
|
||||
account.gold_balance
|
||||
if gold_result
|
||||
else self._to_int(result.get("gold_balance"))
|
||||
)
|
||||
balance_confirmed = bool(
|
||||
baseline_ready
|
||||
and current_gold is not None
|
||||
and baseline is not None
|
||||
and current_gold > baseline
|
||||
)
|
||||
result["supplier_balance_confirmed"] = balance_confirmed
|
||||
|
||||
if balance_confirmed:
|
||||
result.pop("payment_poll_error", None)
|
||||
result["gold_recharged"] = True
|
||||
self._finish_supplier_gold_order(
|
||||
db,
|
||||
task,
|
||||
account,
|
||||
result,
|
||||
amount=amount,
|
||||
status=2,
|
||||
balance_confirmed=True,
|
||||
)
|
||||
return
|
||||
if supplier_error:
|
||||
result["payment_poll_error"] = supplier_error
|
||||
self._schedule_payment_poll(
|
||||
db,
|
||||
task,
|
||||
result,
|
||||
f"供应商直充订单处理中(状态 {status if status is not None else '-'})",
|
||||
f"查询供应商订单失败: {supplier_error}",
|
||||
)
|
||||
except FishFinRechargeError as exc:
|
||||
result["payment_poll_error"] = str(exc)
|
||||
self._schedule_payment_poll(
|
||||
return
|
||||
if code != 200:
|
||||
self._finish_supplier_gold_order(
|
||||
db,
|
||||
task,
|
||||
account,
|
||||
result,
|
||||
f"查询供应商订单失败: {exc}",
|
||||
amount=amount,
|
||||
status=(status if status in {2, 3, 4} else 4),
|
||||
)
|
||||
return
|
||||
if status in {2, 3, 4}:
|
||||
self._finish_supplier_gold_order(
|
||||
db,
|
||||
task,
|
||||
account,
|
||||
result,
|
||||
amount=amount,
|
||||
status=status,
|
||||
)
|
||||
return
|
||||
if gold_error:
|
||||
result["payment_poll_error"] = gold_error
|
||||
else:
|
||||
result.pop("payment_poll_error", None)
|
||||
self._schedule_payment_poll(
|
||||
db,
|
||||
task,
|
||||
result,
|
||||
"供应商直充订单处理中"
|
||||
f"(状态 {status if status is not None else '-'},"
|
||||
f"斗鱼余额 {current_gold if current_gold is not None else '-'})",
|
||||
)
|
||||
|
||||
def _finish_supplier_gold_order(
|
||||
self,
|
||||
@@ -286,8 +355,9 @@ class GoldMixin:
|
||||
*,
|
||||
amount: int,
|
||||
status: int | None,
|
||||
balance_confirmed: bool = False,
|
||||
) -> None:
|
||||
if status == 2:
|
||||
if status == 2 or balance_confirmed:
|
||||
account.updated_at = datetime.now(UTC)
|
||||
record_recharge(
|
||||
db,
|
||||
@@ -300,9 +370,19 @@ class GoldMixin:
|
||||
douyu_uid=str(result.get("douyu_uid") or ""),
|
||||
balance_after=account.gold_balance,
|
||||
actor_id=int(task.created_by),
|
||||
confirmed=False,
|
||||
confirmed=balance_confirmed,
|
||||
)
|
||||
self._mark_task(
|
||||
db,
|
||||
task,
|
||||
"success",
|
||||
(
|
||||
"供应商直充到账成功(斗鱼余额已增加)"
|
||||
if balance_confirmed
|
||||
else "供应商直充成功"
|
||||
),
|
||||
result,
|
||||
)
|
||||
self._mark_task(db, task, "success", "供应商直充成功", result)
|
||||
return
|
||||
self._mark_task(db, task, "failed", "供应商直充失败", result)
|
||||
|
||||
@@ -370,6 +450,21 @@ class GoldMixin:
|
||||
if not recharge_account:
|
||||
raise FishFinRechargeError("账号缺少斗鱼昵称,无法发起供应商直充")
|
||||
|
||||
# 供应商订单可能很快到账;基线必须在下单前读取,否则首轮轮询看到的
|
||||
# 已到账余额会被误当成充值前余额,之后永远等不到“余额增加”。
|
||||
baseline_gold = account.gold_balance
|
||||
baseline_gold_ready = False
|
||||
try:
|
||||
balance_result = self._refresh_account_gold_balance(
|
||||
self._client(cookie, direct=True),
|
||||
account,
|
||||
)
|
||||
baseline_gold = self._to_int(balance_result.get("gold_balance"))
|
||||
baseline_gold_ready = baseline_gold is not None
|
||||
db.commit()
|
||||
except Exception as exc: # noqa: BLE001 供应商下单不能被基线查询阻塞
|
||||
self._push_log("warning", f"供应商直充下单前刷新鱼翅基线失败: {exc}")
|
||||
|
||||
# 首次生成后持久化,网络重试或进程重启都继续查询同一笔订单。
|
||||
order_no = self._supplier_out_order_id(task)
|
||||
task.supplier_out_order_id = order_no
|
||||
@@ -453,6 +548,9 @@ class GoldMixin:
|
||||
"product_id": product_id,
|
||||
"pay_amount": format(pay_amount.normalize(), "f"),
|
||||
"order_type": 0,
|
||||
"baseline_gold_balance": baseline_gold,
|
||||
"baseline_gold_ready": baseline_gold_ready,
|
||||
"baseline_gold_source": "pre_order" if baseline_gold_ready else "",
|
||||
"supplier_code": code,
|
||||
"supplier_order_status": status,
|
||||
"supplier_order": self._supplier_result(order_payload),
|
||||
|
||||
Reference in New Issue
Block a user