perf(douyu): 优化慢查询并修复供应商充值收口

This commit is contained in:
yml
2026-11-21 11:20:58 +08:00
parent 3987464035
commit 9c90a452c5
14 changed files with 435 additions and 69 deletions
+2 -1
View File
@@ -67,7 +67,8 @@
- `capability` 模式只读能力索引,不执行全量候选扫描;
- `legacy` 模式保留旧逻辑,用于快速回滚;
- 能力索引命中后仍只对最终候选做一次实时确认,避免缓存过期直接分配;
- 有 pending 履约需求时,`account_capability_refresh` 作业优先刷新对应商品的过期能力。
- 有 pending 履约需求时,`account_capability_refresh` 作业优先刷新对应商品的过期能力;
作业消息只报告本轮结果,不再每轮执行会扫描全账号的精确剩余量统计。
能力快照默认 15 分钟有效,可用 `ACCOUNT_COMMODITY_STATE_TTL_SECONDS` 调整。
过期能力不会直接参与自动匹配。真正创建租约仍然依赖账号租约唯一约束,
+1 -1
View File
@@ -20,8 +20,8 @@
| `ck_health` | 检测持有 CK 的账号 | 区分 `valid`、`invalid`、`error`;`error` 不覆盖上次有效结论 |
| `ck_relogin` | 对失效 CK 自动重登 | 有退避、次数上限和失败分型;极验/凭证/必须绑手机转人工 |
| `account_overview_refresh` | 分批刷新账号资料和一键查询快照 | 只查 CK 有效且未停用账号;“没查过”优先于“过期重查” |
| `douyu_tasks_retention` | 分批清理斗鱼执行日志 | 默认启用;噪声查询保留 7 天,其余任务保留 90 天,在途任务不删 |
| `points_expire` | 补记积分到期 | 只写账号事件,不修改外部平台 |
| `douyu_tasks_retention` | 清理过期任务日志 | 按任务类型保留期分批删除 |
所有账号级作业都必须排除 `status` 为 `disabled` / `archived` 的账号,并尊重
`auto_managed=False`。停用是“账号不可用”,`auto_managed=False` 是“账号可用但不要自动操作”,
@@ -13,6 +13,7 @@ from web.backend.models import (
User,
)
from web.backend.services.automation.jobs import account_capability_refresh as job
from web.backend.services.automation.scheduler import JobSpec, ensure_jobs, register_job
NOW = datetime.now(UTC).replace(microsecond=0)
@@ -121,3 +122,68 @@ def test_no_pending_task_means_no_refresh(db):
)
assert job.build_select_query(db, automation_job, NOW).all() == []
def test_run_does_not_execute_exact_remaining_count(db, monkeypatch):
db.add(
PlatformFulfillmentTask(
source_system="order_site",
source_order_id="2",
source_order_item_id="2",
internal_sku_key="skin",
commodity_id="C",
execution_status="ready",
binding_status="not_started",
exchange_status="not_started",
paid_at=NOW,
)
)
stale = _account(db, "cap-run-stale")
_state(db, stale, valid_until=NOW - timedelta(minutes=1))
db.commit()
automation_job = AutomationJob(
key=job.JOB_KEY,
title="履约商品能力补充刷新",
interval_seconds=3600,
batch_size=10,
concurrency=3,
)
captured: dict[str, object] = {}
def fake_run_refresh(_db, _job, accounts, **kwargs):
captured["accounts"] = [row.id for row in accounts]
captured["remaining_count"] = kwargs.get("remaining_count")
return {"scanned": len(accounts)}
monkeypatch.setattr(job, "run_refresh_for_accounts", fake_run_refresh)
job.run_account_capability_refresh(db, automation_job)
assert captured["accounts"] == [stale.id]
assert captured["remaining_count"] is None
def test_default_enabled_job_is_created_enabled(db, monkeypatch):
from web.backend.services.automation import scheduler
monkeypatch.setattr(scheduler, "_REGISTRY", {})
register_job(
JobSpec(
key="retention-default-on",
title="默认启用测试",
interval_seconds=3600,
batch_size=10,
concurrency=1,
run=lambda _db, _job: {"scanned": 0},
default_enabled=True,
)
)
ensure_jobs(db)
row = (
db.query(AutomationJob)
.filter(AutomationJob.key == "retention-default-on")
.one()
)
assert row.enabled is True
+10 -2
View File
@@ -409,7 +409,11 @@ class TestScheduler(_Base):
outcomes = run_due_jobs(self.session)
assert outcomes[0]["status"] == "failed"
run = self.session.query(AutomationRun).one()
run = (
self.session.query(AutomationRun)
.filter(AutomationRun.job_key == "boom")
.one()
)
assert run.status == "failed" and "接口挂了" in run.message
def test_next_run_follows_tick_and_counters_persist(self):
@@ -434,7 +438,11 @@ class TestScheduler(_Base):
assert _aware(job.next_run_at) - _aware(job.last_run_at) == timedelta(
seconds=JOB_INTERVAL_SECONDS
)
run = self.session.query(AutomationRun).one()
run = (
self.session.query(AutomationRun)
.filter(AutomationRun.job_key == "probe")
.one()
)
assert (run.scanned, run.valid, run.invalid) == (3, 2, 1)
def test_ensure_jobs_keeps_operator_settings(self):
+51 -2
View File
@@ -41,6 +41,15 @@ class TestDouyuGoldRechargeChannel:
self.session.add(self.task)
self.session.commit()
self.runner = DouyuBatchRunner(self.session, "batch", "create_gold_qr")
self.runner._client = Mock(return_value=Mock())
self.runner._refresh_account_gold_balance = Mock(
return_value={
"gold_balance": 0,
"exchange_balance": 0,
"gold": {"gold": 0},
"exchange_balance_query": {"count": 0},
}
)
os.environ["FISH_FIN_RECHARGE_BASE_URL"] = "https://supplier.example"
os.environ["FISH_FIN_RECHARGE_APP_ID"] = "test-app"
os.environ["FISH_FIN_RECHARGE_APP_SECRET"] = "test-secret"
@@ -48,6 +57,33 @@ class TestDouyuGoldRechargeChannel:
def teardown_method(self):
self.session.close()
def test_payment_dispatcher_routes_supplier_channel_to_supplier_poll(self):
self.account.cookie = "acf_uid=10001"
self.task.result = {
"recharge_channel": "supplier_api",
"buy_num": 30,
"out_order_id": "SUPPLIER-30",
}
self.session.commit()
self.runner._poll_create_gold_supplier_once = Mock()
self.runner._poll_create_gold_qr_once = Mock()
self.runner.poll_payment_task_once(
self.session,
self.task,
self.account,
{},
)
self.runner._poll_create_gold_supplier_once.assert_called_once_with(
self.session,
self.task,
self.account,
"acf_uid=10001",
amount=30,
)
self.runner._poll_create_gold_qr_once.assert_not_called()
def test_wallet_gold_qr_schedules_poll_and_completes_after_balance_grows(
self,
):
@@ -193,7 +229,7 @@ class TestDouyuGoldRechargeChannel:
}
supplier.query_order.return_value = {
"code": 200,
"result": {"order_status": 2, "order_id": "supplier-002"},
"result": {"order_status": 0, "order_id": "supplier-002"},
}
client_class.return_value = supplier
config = {
@@ -216,18 +252,31 @@ class TestDouyuGoldRechargeChannel:
assert self.task.status == "running"
assert self.task.queue_lane == "payment_poll"
assert self.task.result["supplier_order_status"] == 1
assert self.task.result["baseline_gold_balance"] == 0
assert self.task.result["baseline_gold_ready"] is True
assert self.task.result["baseline_gold_source"] == "pre_order"
self.runner._refresh_account_gold_balance = Mock(
return_value={
"gold_balance": 18,
"exchange_balance": 0,
"gold": {"gold": 18},
"exchange_balance_query": {"count": 0},
}
)
self.runner._poll_create_gold_supplier_once(
self.session,
self.task,
self.account,
"acf_uid=10001",
amount=10,
)
self.session.refresh(self.task)
assert self.task.status == "success"
assert self.task.queue_lane == ""
assert self.task.result["supplier_order_status"] == 2
assert self.task.result["supplier_order_status"] == 0
assert self.task.result["supplier_balance_confirmed"] is True
def test_supplier_out_order_id_reuses_persisted_value(self):
self.task.supplier_out_order_id = "DYGFHISTORICAL-001"
+61 -1
View File
@@ -29,6 +29,7 @@ from web.backend.models import (
AccountEvent,
AccountLease,
AuditLog,
AutomationJob,
DouyuExternalOrder,
HuyaExternalOrder,
PlatformFulfillmentTask,
@@ -40,7 +41,7 @@ TESTS_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = TESTS_DIR.parent
VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions"
HEAD_REVISION = "20261120_0103"
HEAD_REVISION = "20261121_0104"
# 迁移新增、但不声明在模型里的复合查询索引。
EXTRA_INDEXES = (
@@ -254,6 +255,65 @@ class TestMigrationSmoke:
with admin_engine.begin() as conn:
conn.execute(text(f"DROP DATABASE IF EXISTS `{mig_db}`;"))
def test_0104_enables_retention_and_adds_huya_status_index(self):
creds = get_mysql_credentials()
mig_db = f"douyu_login_mig_{secrets.token_hex(4)}"
mig_url = (
f"mysql+pymysql://{quote_plus(creds['user'])}:"
f"{quote_plus(creds['password'])}@{creds['host']}:{creds['port']}/"
f"{mig_db}?charset=utf8mb4"
)
admin_engine = get_test_engine()
mig_engine = create_engine(mig_url)
original_url = database_module.DATABASE_URL
with admin_engine.begin() as conn:
conn.execute(
text(
f"CREATE DATABASE `{mig_db}` "
"CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;"
)
)
try:
database_module.DATABASE_URL = mig_url
cfg = _migration_config()
cfg.set_main_option("sqlalchemy.url", mig_url)
command.upgrade(cfg, "20261120_0103")
with Session(mig_engine) as session:
session.add(
AutomationJob(
key="douyu_tasks_retention",
title="任务日志清理",
enabled=False,
interval_seconds=24 * 3600,
run_interval_seconds=300,
batch_size=2000,
concurrency=1,
)
)
session.commit()
command.upgrade(cfg, "head")
with Session(mig_engine) as session:
job = (
session.query(AutomationJob)
.filter(AutomationJob.key == "douyu_tasks_retention")
.one()
)
assert job.enabled is True
assert job.next_run_at is not None
indexes = {
row["name"]
for row in inspect(mig_engine).get_indexes("huya_tasks")
if row.get("name")
}
assert "ix_huya_tasks_account_status_id" in indexes
finally:
database_module.DATABASE_URL = original_url
mig_engine.dispose()
with admin_engine.begin() as conn:
conn.execute(text(f"DROP DATABASE IF EXISTS `{mig_db}`;"))
def test_0077_converts_super_admin_membership_to_overlay(self):
creds = get_mysql_credentials()
mig_db = f"douyu_login_mig_{secrets.token_hex(4)}"
@@ -0,0 +1,66 @@
"""启用任务日志清理并补虎牙任务状态索引。"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20261121_0104"
down_revision = "20261120_0103"
branch_labels = None
depends_on = None
def _index_exists(table_name: str, index_name: str) -> bool:
bind = op.get_bind()
return bool(
bind.execute(
sa.text(
"""
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = :table_name
AND index_name = :index_name
"""
),
{"table_name": table_name, "index_name": index_name},
).scalar()
)
def upgrade() -> None:
if not _index_exists("huya_tasks", "ix_huya_tasks_account_status_id"):
op.create_index(
"ix_huya_tasks_account_status_id",
"huya_tasks",
["account_id", "status", "id"],
unique=False,
)
op.execute(
sa.text(
"""
UPDATE automation_jobs
SET enabled = 1,
next_run_at = COALESCE(next_run_at, UTC_TIMESTAMP()),
updated_at = UTC_TIMESTAMP()
WHERE `key` = 'douyu_tasks_retention'
"""
)
)
def downgrade() -> None:
op.execute(
sa.text(
"""
UPDATE automation_jobs
SET enabled = 0,
next_run_at = NULL,
updated_at = UTC_TIMESTAMP()
WHERE `key` = 'douyu_tasks_retention'
"""
)
)
if _index_exists("huya_tasks", "ix_huya_tasks_account_status_id"):
op.drop_index("ix_huya_tasks_account_status_id", table_name="huya_tasks")
+5
View File
@@ -1479,6 +1479,11 @@ class HuyaTask(Base):
account: Mapped[HuyaAccount] = relationship("HuyaAccount", back_populates="tasks")
__table_args__ = (
# 客服工作台按归属账号汇总任务状态时走覆盖索引,避免回表排序。
Index("ix_huya_tasks_account_status_id", "account_id", "status", "id"),
)
class HuyaWorkbenchAccount(Base):
"""用户在虎牙指定工作台中启用的账号。"""
@@ -114,6 +114,7 @@ def register_default_jobs() -> None:
batch_size=2000,
concurrency=1,
run=run_douyu_tasks_retention,
default_enabled=True,
)
)
register_job(
@@ -84,14 +84,6 @@ def build_select_query(
)
def count_remaining(
db: Session,
*,
now: datetime | None = None,
) -> int:
return _base_query(db, now or datetime.now(UTC)).count()
def run_account_capability_refresh(
db: Session,
job: AutomationJob,
@@ -103,7 +95,9 @@ def run_account_capability_refresh(
job,
accounts,
now=now,
remaining_count=count_remaining(db, now=now),
# 精确剩余量要完整评估 accounts × commodity EXISTS,生产实测单次约
# 1.3 秒、扫描 38 万行;它只用于展示,直接省略,避免每轮制造慢查询。
remaining_count=None,
source="capability_demand_refresh",
log_label="capability",
empty_message="当前履约商品没有到期待刷新的账号能力",
@@ -113,6 +107,5 @@ def run_account_capability_refresh(
__all__ = [
"JOB_KEY",
"build_select_query",
"count_remaining",
"run_account_capability_refresh",
]
@@ -34,7 +34,8 @@
- 不删行、只删「过期的整行」:不做「把 result 清空保留行」这种半吊子操作,
因为部分前端派生**明确要求 result 非空**,清空等于制造「有行但读不出状态」的
更隐蔽的坏状态;
- 作业**默认关闭**(`ensure_jobs` 建行时 enabled=False),由运维在页面上打开。
- 作业**默认启用**(`ensure_jobs` 建行时 enabled=True),避免执行日志无限增长;
运维仍可在页面上关闭。
"""
from __future__ import annotations
+5 -2
View File
@@ -4,7 +4,8 @@
自动化按「渐进上线」推进,因此:
- 调度器总开关 ``AUTOMATION_ENABLED`` **默认关闭**;
- 每个作业自己还有开关,**默认关闭**,在 ``/automation`` 页面打开;
- 每个作业自己还有开关,**默认关闭**;少数维护型作业可通过
``JobSpec.default_enabled`` 在首次建行时默认启用,之后仍完全尊重页面配置;
- 作业状态全部落库(``automation_jobs`` / ``automation_runs``),进程重启不丢
调度进度,也不会因为重启而重复跑一轮。
@@ -65,6 +66,8 @@ class JobSpec:
#: **多久跑一次**(调度频率)。默认 5 分钟,与改动前的实际行为一致。
#: ⚠️ 必须放在**有默认值**的位置(dataclass 不允许默认值字段排在无默认值之前)。
run_interval_seconds: int = 300
#: 新库首次创建作业行时是否默认启用。已有行永远尊重运维配置。
default_enabled: bool = False
_REGISTRY: dict[str, JobSpec] = {}
@@ -120,7 +123,7 @@ def ensure_jobs(db: Session) -> None:
AutomationJob(
key=spec.key,
title=spec.title,
enabled=False,
enabled=spec.default_enabled,
interval_seconds=spec.interval_seconds,
run_interval_seconds=spec.run_interval_seconds,
batch_size=spec.batch_size,
+16 -1
View File
@@ -63,7 +63,22 @@ class DouyuBatchRunner(
)
return
if task.task_type == "create_gold_qr":
self._poll_create_gold_qr_once(db, task, account, cookie, config)
if str(result.get("recharge_channel") or "") == "supplier_api":
amount = int(
result.get("buy_num")
or self._task_payload(task).get("amount")
or self._task_payload(task).get("gold_amount")
or 1
)
self._poll_create_gold_supplier_once(
db,
task,
account,
cookie,
amount=amount,
)
else:
self._poll_create_gold_qr_once(db, task, account, cookie, config)
elif task.task_type == "create_elite_qr":
self._poll_create_elite_qr_once(db, task, account, cookie, config)
elif task.task_type == "create_esports_qr":
+146 -48
View File
@@ -197,9 +197,11 @@ class GoldMixin:
db: Session,
task: DouyuTask,
account: Account,
cookie: str,
*,
amount: int,
) -> None:
"""查询一次供应商直充订单状态。"""
"""同时核对供应商订单和斗鱼余额,任一证明到账即可收口。"""
db.refresh(task)
if task.status in {"success", "failed", "error", "stopped"}:
return
@@ -208,6 +210,20 @@ class GoldMixin:
order_no = str(result["out_order_id"])
result["payment_polling"] = True
result["payment_poll_count"] = int(result.get("payment_poll_count") or 0) + 1
code: int | None = None
status: int | None = None
supplier_error = ""
gold_result: dict[str, Any] = {}
gold_error = ""
try:
gold_result = self._refresh_account_gold_balance(
self._client(cookie, direct=True),
account,
)
except Exception as exc: # noqa: BLE001 余额核对失败仍继续查供应商
gold_error = str(exc)
try:
client = FishFinRechargeClient(FishFinRechargeConfig.from_env())
payload = client.query_order(order_no)
@@ -220,62 +236,115 @@ class GoldMixin:
"supplier_order": self._supplier_result(payload),
}
)
# 查询期间供应商回调可能已经把任务收口。重新锁行后再决定是否写入,
# 避免长时间 HTTP 调用结束后用旧状态覆盖回调结果。
db.rollback()
locked = (
db.query(DouyuTask)
.filter(
DouyuTask.id == task_id,
DouyuTask.status == "running",
DouyuTask.queue_lane == "payment_poll",
)
.with_for_update()
.first()
except Exception as exc: # noqa: BLE001 供应商查询异常由持久调度继续重试
supplier_error = str(exc)
# 查询期间供应商回调可能已经把任务收口。重新锁行后再决定是否写入,
# 避免长时间 HTTP 调用结束后用旧状态覆盖回调结果。
db.rollback()
locked = (
db.query(DouyuTask)
.filter(
DouyuTask.id == task_id,
DouyuTask.status == "running",
DouyuTask.queue_lane == "payment_poll",
)
if locked is None:
return
task = locked
locked_account = (
db.query(Account).filter(Account.id == task.account_id).first()
.with_for_update()
.first()
)
if locked is None:
return
task = locked
locked_account = db.query(Account).filter(Account.id == task.account_id).first()
if locked_account is None:
self._mark_task(
db,
task,
"failed",
"账号不存在,无法继续核对供应商充值到账",
result,
)
if locked_account is not None:
account = locked_account
if code != 200:
self._finish_supplier_gold_order(
db,
task,
account,
result,
amount=amount,
status=(status if status in {2, 3, 4} else 4),
)
return
if status in {2, 3, 4}:
self._finish_supplier_gold_order(
db,
task,
account,
result,
amount=amount,
status=status,
)
return
return
account = locked_account
if gold_result:
result.update(gold_result)
account.gold_balance = self._to_int(gold_result.get("gold_balance"))
account.exchange_balance = self._to_int(gold_result.get("exchange_balance"))
account.updated_at = datetime.now(UTC)
baseline = self._to_int(result.get("baseline_gold_balance"))
baseline_ready = bool(
result.get("baseline_gold_ready")
and result.get("baseline_gold_source") == "pre_order"
and baseline is not None
)
current_gold = (
account.gold_balance
if gold_result
else self._to_int(result.get("gold_balance"))
)
balance_confirmed = bool(
baseline_ready
and current_gold is not None
and baseline is not None
and current_gold > baseline
)
result["supplier_balance_confirmed"] = balance_confirmed
if balance_confirmed:
result.pop("payment_poll_error", None)
result["gold_recharged"] = True
self._finish_supplier_gold_order(
db,
task,
account,
result,
amount=amount,
status=2,
balance_confirmed=True,
)
return
if supplier_error:
result["payment_poll_error"] = supplier_error
self._schedule_payment_poll(
db,
task,
result,
f"供应商直充订单处理中(状态 {status if status is not None else '-'})",
f"查询供应商订单失败: {supplier_error}",
)
except FishFinRechargeError as exc:
result["payment_poll_error"] = str(exc)
self._schedule_payment_poll(
return
if code != 200:
self._finish_supplier_gold_order(
db,
task,
account,
result,
f"查询供应商订单失败: {exc}",
amount=amount,
status=(status if status in {2, 3, 4} else 4),
)
return
if status in {2, 3, 4}:
self._finish_supplier_gold_order(
db,
task,
account,
result,
amount=amount,
status=status,
)
return
if gold_error:
result["payment_poll_error"] = gold_error
else:
result.pop("payment_poll_error", None)
self._schedule_payment_poll(
db,
task,
result,
"供应商直充订单处理中"
f"(状态 {status if status is not None else '-'},"
f"斗鱼余额 {current_gold if current_gold is not None else '-'})",
)
def _finish_supplier_gold_order(
self,
@@ -286,8 +355,9 @@ class GoldMixin:
*,
amount: int,
status: int | None,
balance_confirmed: bool = False,
) -> None:
if status == 2:
if status == 2 or balance_confirmed:
account.updated_at = datetime.now(UTC)
record_recharge(
db,
@@ -300,9 +370,19 @@ class GoldMixin:
douyu_uid=str(result.get("douyu_uid") or ""),
balance_after=account.gold_balance,
actor_id=int(task.created_by),
confirmed=False,
confirmed=balance_confirmed,
)
self._mark_task(
db,
task,
"success",
(
"供应商直充到账成功(斗鱼余额已增加)"
if balance_confirmed
else "供应商直充成功"
),
result,
)
self._mark_task(db, task, "success", "供应商直充成功", result)
return
self._mark_task(db, task, "failed", "供应商直充失败", result)
@@ -370,6 +450,21 @@ class GoldMixin:
if not recharge_account:
raise FishFinRechargeError("账号缺少斗鱼昵称,无法发起供应商直充")
# 供应商订单可能很快到账;基线必须在下单前读取,否则首轮轮询看到的
# 已到账余额会被误当成充值前余额,之后永远等不到“余额增加”。
baseline_gold = account.gold_balance
baseline_gold_ready = False
try:
balance_result = self._refresh_account_gold_balance(
self._client(cookie, direct=True),
account,
)
baseline_gold = self._to_int(balance_result.get("gold_balance"))
baseline_gold_ready = baseline_gold is not None
db.commit()
except Exception as exc: # noqa: BLE001 供应商下单不能被基线查询阻塞
self._push_log("warning", f"供应商直充下单前刷新鱼翅基线失败: {exc}")
# 首次生成后持久化,网络重试或进程重启都继续查询同一笔订单。
order_no = self._supplier_out_order_id(task)
task.supplier_out_order_id = order_no
@@ -453,6 +548,9 @@ class GoldMixin:
"product_id": product_id,
"pay_amount": format(pay_amount.normalize(), "f"),
"order_type": 0,
"baseline_gold_balance": baseline_gold,
"baseline_gold_ready": baseline_gold_ready,
"baseline_gold_source": "pre_order" if baseline_gold_ready else "",
"supplier_code": code,
"supplier_order_status": status,
"supplier_order": self._supplier_result(order_payload),