Files
live-hub-py/tests/test_douyu_tasks_retention.py
T

252 lines
10 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""S5b:`douyu_tasks` 保留策略。
守住的核心不是「删得掉」,而是**删得对**:
- 噪声查询按短窗口删;
- **业务事实与前端状态派生依赖的查询按长窗口删**(删早了工作台账号行会变「未查」);
- **未知类型默认长保留**(新增业务类型不该因为「忘了加进某个列表」而被误删);
- **在途任务绝不删**(批次线程正拿着那一行)。
"""
from datetime import UTC, datetime, timedelta
from sqlalchemy import func
from tests.support.db import get_test_engine, make_test_session
from web.backend.models import Account, AutomationJob, DouyuTask, User
from web.backend.services.automation.jobs import douyu_tasks_retention as retention
NOW = datetime.now(UTC)
class _Base:
def setup_method(self):
self.engine = get_test_engine()
self.session = make_test_session(self.engine)
self.session.query(DouyuTask).delete()
self.session.query(AutomationJob).filter(
AutomationJob.key == retention.JOB_KEY
).delete()
self.session.commit()
self.user = User(username="ret-admin", password_hash="h", role="super_admin")
self.session.add(self.user)
self.session.flush()
self.account = Account(
username="ret-acc", password="p", email="r@e.com", email_password="ep"
)
self.session.add(self.account)
self.session.commit()
# 记下 id:`run()` 会 expunge_all(),之后不能再碰 self.account 的属性
# (commit 会 expire 属性,detach 后再访问就是 DetachedInstanceError)。
self.account_id = self.account.id
self.user_id = self.user.id
def teardown_method(self):
self.session.close()
def make_task(
self,
task_type: str,
*,
age_days: float,
status: str = "success",
result: dict | None = None,
) -> DouyuTask:
task = DouyuTask(
batch_id="ret-batch",
account_id=self.account_id,
task_type=task_type,
status=status,
message="m",
result=result if result is not None else {"payload": {}},
created_by=self.user_id,
created_at=NOW - timedelta(days=age_days),
)
self.session.add(task)
self.session.commit()
return task
def job(self, **config) -> AutomationJob:
"""取(或建)作业行。
幂等:`automation_jobs` 的主键就是 `key`,一个用例里跑两次不能插两行。
"""
job = self.session.get(AutomationJob, retention.JOB_KEY)
if job is None:
job = AutomationJob(
key=retention.JOB_KEY,
title="任务日志清理",
enabled=True,
interval_seconds=24 * 3600,
batch_size=100,
concurrency=1,
config=config or None,
updated_at=NOW,
)
self.session.add(job)
job.config = config or None
job.batch_size = 100
self.session.commit()
return job
def run(self, **config) -> dict:
out = retention.run_douyu_tasks_retention(self.session, self.job(**config))
# 作业用**裸 SQL** 删行,ORM 的 identity map 不知道行没了;
# 不 expunge 的话后面访问这些对象会抛 ObjectDeletedError(而不是干净地查不到)。
self.session.expunge_all()
return out
def remaining(self) -> set[tuple[str, int]]:
return {(t.task_type, t.id) for t in self.session.query(DouyuTask).all()}
def count(self) -> int:
return int(self.session.query(func.count(DouyuTask.id)).scalar() or 0)
class TestRetention(_Base):
def test_noise_query_deleted_after_short_window(self):
self.make_task("query_limited_goods", age_days=10)
self.make_task("query_change_bind_time", age_days=10)
out = self.run(noise_days=7, business_days=90)
assert out["deleted"] == 2
assert out["deleted_noise"] == 2
assert self.remaining() == set()
def test_noise_query_kept_within_short_window(self):
self.make_task("query_limited_goods", age_days=3)
out = self.run(noise_days=7, business_days=90)
assert out["deleted"] == 0
assert self.count() == 1
def test_business_fact_kept_past_short_window(self):
"""业务事实不该被 7 天窗口带走 —— 这是分档的全部意义。"""
for task_type in (
"exchange_goods",
"create_gold_qr",
"confirm_bind",
"exchange_xpd_goods",
):
self.make_task(task_type, age_days=30)
out = self.run(noise_days=7, business_days=90)
assert out["deleted"] == 0
assert self.count() == 4
def test_business_fact_deleted_after_long_window(self):
self.make_task("exchange_goods", age_days=100)
out = self.run(noise_days=7, business_days=90)
assert out["deleted"] == 1
assert out["deleted_rest"] == 1
def test_state_derivation_queries_kept_longer(self):
"""工作台从这些查询的 result 反推账号当前状态,删早了显示会变「未查」。"""
for task_type in (
"query_xpd_bind_info",
"query_xpd_role",
"query_esports_game_name",
"query_game_name",
"prepare_esports_bind",
"get_esports_bind_qr",
):
self.make_task(task_type, age_days=30)
out = self.run(noise_days=7, business_days=90)
assert out["deleted"] == 0, f"状态派生依赖的任务被误删:{self.remaining()}"
def test_unknown_task_type_defaults_to_long_retention(self):
"""新增的任务类型默认长保留 —— 绝不能因为「没列进名单」而被删掉。"""
self.make_task("some_brand_new_task_type", age_days=30)
out = self.run(noise_days=7, business_days=90)
assert out["deleted"] == 0
assert self.count() == 1
# 但只要过了长窗口就该被清掉(否则未知类型会永久堆积)
self.make_task("another_new_type", age_days=200)
out = self.run(noise_days=7, business_days=90)
assert out["deleted"] == 1
def test_in_flight_tasks_never_deleted(self):
"""批次线程可能正拿着这些行,无论多老都不能删。"""
for status in ("planned", "pending", "running"):
self.make_task("query_limited_goods", age_days=999, status=status)
self.make_task("query_limited_goods", age_days=999, status="success")
out = self.run(noise_days=7, business_days=90)
assert out["deleted"] == 1
statuses = {t.status for t in self.session.query(DouyuTask).all()}
assert statuses == {"planned", "pending", "running"}
def test_config_windows_are_honoured(self):
self.make_task("query_limited_goods", age_days=2)
out = self.run(noise_days=1, business_days=90)
assert out["deleted"] == 1
assert out["noise_days"] == 1
def test_invalid_config_falls_back_to_defaults(self):
out = self.run(noise_days=0, business_days="abc")
assert out["noise_days"] == retention.DEFAULT_NOISE_DAYS
assert out["business_days"] == retention.DEFAULT_BUSINESS_DAYS
def test_result_payload_is_dropped_with_the_row(self):
"""删整行(而不是只清 result)—— 避免「有行但读不出状态」的隐蔽坏状态。"""
task = self.make_task(
"query_limited_goods",
age_days=30,
result={"limited_goods": [{"a": 1}] * 100},
)
task_id = task.id # 先取:run() 会 expunge_all,之后对象就 detached 了
self.run(noise_days=7, business_days=90)
assert self.session.get(DouyuTask, task_id) is None
def test_batch_budget_is_reported_not_overclaimed(self):
"""预算用尽只报「预算用光」,不谎称「仍有积压」。
预算在上一档用光时,下一档根本没轮到执行 —— 这时无法区分「没得删」和
「没轮到」,所以只给提示。
"""
for _ in range(5):
self.make_task("query_limited_goods", age_days=30)
out = self.run(noise_days=7, business_days=90, max_batches=1)
# batch_size=100(job 默认),5 条一轮删完(不足一批就 break),预算正好用光
assert out["batches"] == 1
assert out["deleted"] == 5
assert out["batch_budget_exhausted"] is True
def test_budget_not_exhausted_when_work_runs_out(self):
self.make_task("query_limited_goods", age_days=30)
out = self.run(noise_days=7, business_days=90, max_batches=50)
assert out["batch_budget_exhausted"] is False
class TestClassification(_Base):
def test_noise_list_has_no_duplicates(self):
assert len(retention.NOISE_TASK_TYPES) == len(set(retention.NOISE_TASK_TYPES))
def test_noise_list_has_no_business_types(self):
business = {
"exchange_goods",
"exchange_esports_goods",
"exchange_xpd_goods",
"pay_locked_order",
"lock_goods",
"create_gold_qr",
"donate_elite_gift",
"donate_esports_chicken_gift",
"donate_esports_firework_gift",
"confirm_bind",
"confirm_esports_bind",
"confirm_xpd_bind",
"create_elite_qr",
"create_esports_qr",
}
overlap = business & set(retention.NOISE_TASK_TYPES)
assert not overlap, f"业务类型被当成噪声了:{overlap}"
def test_noise_types_exist_in_supported_types(self):
"""噪声名单里的类型必须都是真实存在的任务类型(防拼写错误导致永不生效)。"""
from web.backend.services.douyu_service import SUPPORTED_DOUYU_TASK_TYPES
known = set(SUPPORTED_DOUYU_TASK_TYPES) | set(
retention.RETIRED_NOISE_TASK_TYPES
)
unknown = set(retention.NOISE_TASK_TYPES) - known
assert not unknown, f"噪声名单里有不存在的任务类型:{unknown}"