perf(douyu): 优化慢查询并修复供应商充值收口

This commit is contained in:
yml
2026-11-21 11:20:58 +08:00
parent 3987464035
commit 9c90a452c5
14 changed files with 435 additions and 69 deletions
@@ -13,6 +13,7 @@ from web.backend.models import (
User,
)
from web.backend.services.automation.jobs import account_capability_refresh as job
from web.backend.services.automation.scheduler import JobSpec, ensure_jobs, register_job
NOW = datetime.now(UTC).replace(microsecond=0)
@@ -121,3 +122,68 @@ def test_no_pending_task_means_no_refresh(db):
)
assert job.build_select_query(db, automation_job, NOW).all() == []
def test_run_does_not_execute_exact_remaining_count(db, monkeypatch):
db.add(
PlatformFulfillmentTask(
source_system="order_site",
source_order_id="2",
source_order_item_id="2",
internal_sku_key="skin",
commodity_id="C",
execution_status="ready",
binding_status="not_started",
exchange_status="not_started",
paid_at=NOW,
)
)
stale = _account(db, "cap-run-stale")
_state(db, stale, valid_until=NOW - timedelta(minutes=1))
db.commit()
automation_job = AutomationJob(
key=job.JOB_KEY,
title="履约商品能力补充刷新",
interval_seconds=3600,
batch_size=10,
concurrency=3,
)
captured: dict[str, object] = {}
def fake_run_refresh(_db, _job, accounts, **kwargs):
captured["accounts"] = [row.id for row in accounts]
captured["remaining_count"] = kwargs.get("remaining_count")
return {"scanned": len(accounts)}
monkeypatch.setattr(job, "run_refresh_for_accounts", fake_run_refresh)
job.run_account_capability_refresh(db, automation_job)
assert captured["accounts"] == [stale.id]
assert captured["remaining_count"] is None
def test_default_enabled_job_is_created_enabled(db, monkeypatch):
from web.backend.services.automation import scheduler
monkeypatch.setattr(scheduler, "_REGISTRY", {})
register_job(
JobSpec(
key="retention-default-on",
title="默认启用测试",
interval_seconds=3600,
batch_size=10,
concurrency=1,
run=lambda _db, _job: {"scanned": 0},
default_enabled=True,
)
)
ensure_jobs(db)
row = (
db.query(AutomationJob)
.filter(AutomationJob.key == "retention-default-on")
.one()
)
assert row.enabled is True
+10 -2
View File
@@ -409,7 +409,11 @@ class TestScheduler(_Base):
outcomes = run_due_jobs(self.session)
assert outcomes[0]["status"] == "failed"
run = self.session.query(AutomationRun).one()
run = (
self.session.query(AutomationRun)
.filter(AutomationRun.job_key == "boom")
.one()
)
assert run.status == "failed" and "接口挂了" in run.message
def test_next_run_follows_tick_and_counters_persist(self):
@@ -434,7 +438,11 @@ class TestScheduler(_Base):
assert _aware(job.next_run_at) - _aware(job.last_run_at) == timedelta(
seconds=JOB_INTERVAL_SECONDS
)
run = self.session.query(AutomationRun).one()
run = (
self.session.query(AutomationRun)
.filter(AutomationRun.job_key == "probe")
.one()
)
assert (run.scanned, run.valid, run.invalid) == (3, 2, 1)
def test_ensure_jobs_keeps_operator_settings(self):
+51 -2
View File
@@ -41,6 +41,15 @@ class TestDouyuGoldRechargeChannel:
self.session.add(self.task)
self.session.commit()
self.runner = DouyuBatchRunner(self.session, "batch", "create_gold_qr")
self.runner._client = Mock(return_value=Mock())
self.runner._refresh_account_gold_balance = Mock(
return_value={
"gold_balance": 0,
"exchange_balance": 0,
"gold": {"gold": 0},
"exchange_balance_query": {"count": 0},
}
)
os.environ["FISH_FIN_RECHARGE_BASE_URL"] = "https://supplier.example"
os.environ["FISH_FIN_RECHARGE_APP_ID"] = "test-app"
os.environ["FISH_FIN_RECHARGE_APP_SECRET"] = "test-secret"
@@ -48,6 +57,33 @@ class TestDouyuGoldRechargeChannel:
def teardown_method(self):
self.session.close()
def test_payment_dispatcher_routes_supplier_channel_to_supplier_poll(self):
self.account.cookie = "acf_uid=10001"
self.task.result = {
"recharge_channel": "supplier_api",
"buy_num": 30,
"out_order_id": "SUPPLIER-30",
}
self.session.commit()
self.runner._poll_create_gold_supplier_once = Mock()
self.runner._poll_create_gold_qr_once = Mock()
self.runner.poll_payment_task_once(
self.session,
self.task,
self.account,
{},
)
self.runner._poll_create_gold_supplier_once.assert_called_once_with(
self.session,
self.task,
self.account,
"acf_uid=10001",
amount=30,
)
self.runner._poll_create_gold_qr_once.assert_not_called()
def test_wallet_gold_qr_schedules_poll_and_completes_after_balance_grows(
self,
):
@@ -193,7 +229,7 @@ class TestDouyuGoldRechargeChannel:
}
supplier.query_order.return_value = {
"code": 200,
"result": {"order_status": 2, "order_id": "supplier-002"},
"result": {"order_status": 0, "order_id": "supplier-002"},
}
client_class.return_value = supplier
config = {
@@ -216,18 +252,31 @@ class TestDouyuGoldRechargeChannel:
assert self.task.status == "running"
assert self.task.queue_lane == "payment_poll"
assert self.task.result["supplier_order_status"] == 1
assert self.task.result["baseline_gold_balance"] == 0
assert self.task.result["baseline_gold_ready"] is True
assert self.task.result["baseline_gold_source"] == "pre_order"
self.runner._refresh_account_gold_balance = Mock(
return_value={
"gold_balance": 18,
"exchange_balance": 0,
"gold": {"gold": 18},
"exchange_balance_query": {"count": 0},
}
)
self.runner._poll_create_gold_supplier_once(
self.session,
self.task,
self.account,
"acf_uid=10001",
amount=10,
)
self.session.refresh(self.task)
assert self.task.status == "success"
assert self.task.queue_lane == ""
assert self.task.result["supplier_order_status"] == 2
assert self.task.result["supplier_order_status"] == 0
assert self.task.result["supplier_balance_confirmed"] is True
def test_supplier_out_order_id_reuses_persisted_value(self):
self.task.supplier_out_order_id = "DYGFHISTORICAL-001"
+61 -1
View File
@@ -29,6 +29,7 @@ from web.backend.models import (
AccountEvent,
AccountLease,
AuditLog,
AutomationJob,
DouyuExternalOrder,
HuyaExternalOrder,
PlatformFulfillmentTask,
@@ -40,7 +41,7 @@ TESTS_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = TESTS_DIR.parent
VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions"
HEAD_REVISION = "20261120_0103"
HEAD_REVISION = "20261121_0104"
# 迁移新增、但不声明在模型里的复合查询索引。
EXTRA_INDEXES = (
@@ -254,6 +255,65 @@ class TestMigrationSmoke:
with admin_engine.begin() as conn:
conn.execute(text(f"DROP DATABASE IF EXISTS `{mig_db}`;"))
def test_0104_enables_retention_and_adds_huya_status_index(self):
creds = get_mysql_credentials()
mig_db = f"douyu_login_mig_{secrets.token_hex(4)}"
mig_url = (
f"mysql+pymysql://{quote_plus(creds['user'])}:"
f"{quote_plus(creds['password'])}@{creds['host']}:{creds['port']}/"
f"{mig_db}?charset=utf8mb4"
)
admin_engine = get_test_engine()
mig_engine = create_engine(mig_url)
original_url = database_module.DATABASE_URL
with admin_engine.begin() as conn:
conn.execute(
text(
f"CREATE DATABASE `{mig_db}` "
"CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;"
)
)
try:
database_module.DATABASE_URL = mig_url
cfg = _migration_config()
cfg.set_main_option("sqlalchemy.url", mig_url)
command.upgrade(cfg, "20261120_0103")
with Session(mig_engine) as session:
session.add(
AutomationJob(
key="douyu_tasks_retention",
title="任务日志清理",
enabled=False,
interval_seconds=24 * 3600,
run_interval_seconds=300,
batch_size=2000,
concurrency=1,
)
)
session.commit()
command.upgrade(cfg, "head")
with Session(mig_engine) as session:
job = (
session.query(AutomationJob)
.filter(AutomationJob.key == "douyu_tasks_retention")
.one()
)
assert job.enabled is True
assert job.next_run_at is not None
indexes = {
row["name"]
for row in inspect(mig_engine).get_indexes("huya_tasks")
if row.get("name")
}
assert "ix_huya_tasks_account_status_id" in indexes
finally:
database_module.DATABASE_URL = original_url
mig_engine.dispose()
with admin_engine.begin() as conn:
conn.execute(text(f"DROP DATABASE IF EXISTS `{mig_db}`;"))
def test_0077_converts_super_admin_membership_to_overlay(self):
creds = get_mysql_credentials()
mig_db = f"douyu_login_mig_{secrets.token_hex(4)}"