252 lines
10 KiB
Python
252 lines
10 KiB
Python
"""S5b:`douyu_tasks` 保留策略。
|
||
|
||
守住的核心不是「删得掉」,而是**删得对**:
|
||
|
||
- 噪声查询按短窗口删;
|
||
- **业务事实与前端状态派生依赖的查询按长窗口删**(删早了工作台账号行会变「未查」);
|
||
- **未知类型默认长保留**(新增业务类型不该因为「忘了加进某个列表」而被误删);
|
||
- **在途任务绝不删**(批次线程正拿着那一行)。
|
||
"""
|
||
|
||
from datetime import UTC, datetime, timedelta
|
||
|
||
from sqlalchemy import func
|
||
|
||
from tests.support.db import get_test_engine, make_test_session
|
||
from web.backend.models import Account, AutomationJob, DouyuTask, User
|
||
from web.backend.services.automation.jobs import douyu_tasks_retention as retention
|
||
|
||
NOW = datetime.now(UTC)
|
||
|
||
|
||
class _Base:
|
||
def setup_method(self):
|
||
self.engine = get_test_engine()
|
||
self.session = make_test_session(self.engine)
|
||
self.session.query(DouyuTask).delete()
|
||
self.session.query(AutomationJob).filter(
|
||
AutomationJob.key == retention.JOB_KEY
|
||
).delete()
|
||
self.session.commit()
|
||
|
||
self.user = User(username="ret-admin", password_hash="h", role="super_admin")
|
||
self.session.add(self.user)
|
||
self.session.flush()
|
||
self.account = Account(
|
||
username="ret-acc", password="p", email="r@e.com", email_password="ep"
|
||
)
|
||
self.session.add(self.account)
|
||
self.session.commit()
|
||
# 记下 id:`run()` 会 expunge_all(),之后不能再碰 self.account 的属性
|
||
# (commit 会 expire 属性,detach 后再访问就是 DetachedInstanceError)。
|
||
self.account_id = self.account.id
|
||
self.user_id = self.user.id
|
||
|
||
def teardown_method(self):
|
||
self.session.close()
|
||
|
||
def make_task(
|
||
self,
|
||
task_type: str,
|
||
*,
|
||
age_days: float,
|
||
status: str = "success",
|
||
result: dict | None = None,
|
||
) -> DouyuTask:
|
||
task = DouyuTask(
|
||
batch_id="ret-batch",
|
||
account_id=self.account_id,
|
||
task_type=task_type,
|
||
status=status,
|
||
message="m",
|
||
result=result if result is not None else {"payload": {}},
|
||
created_by=self.user_id,
|
||
created_at=NOW - timedelta(days=age_days),
|
||
)
|
||
self.session.add(task)
|
||
self.session.commit()
|
||
return task
|
||
|
||
def job(self, **config) -> AutomationJob:
|
||
"""取(或建)作业行。
|
||
|
||
幂等:`automation_jobs` 的主键就是 `key`,一个用例里跑两次不能插两行。
|
||
"""
|
||
job = self.session.get(AutomationJob, retention.JOB_KEY)
|
||
if job is None:
|
||
job = AutomationJob(
|
||
key=retention.JOB_KEY,
|
||
title="任务日志清理",
|
||
enabled=True,
|
||
interval_seconds=24 * 3600,
|
||
batch_size=100,
|
||
concurrency=1,
|
||
config=config or None,
|
||
updated_at=NOW,
|
||
)
|
||
self.session.add(job)
|
||
job.config = config or None
|
||
job.batch_size = 100
|
||
self.session.commit()
|
||
return job
|
||
|
||
def run(self, **config) -> dict:
|
||
out = retention.run_douyu_tasks_retention(self.session, self.job(**config))
|
||
# 作业用**裸 SQL** 删行,ORM 的 identity map 不知道行没了;
|
||
# 不 expunge 的话后面访问这些对象会抛 ObjectDeletedError(而不是干净地查不到)。
|
||
self.session.expunge_all()
|
||
return out
|
||
|
||
def remaining(self) -> set[tuple[str, int]]:
|
||
return {(t.task_type, t.id) for t in self.session.query(DouyuTask).all()}
|
||
|
||
def count(self) -> int:
|
||
return int(self.session.query(func.count(DouyuTask.id)).scalar() or 0)
|
||
|
||
|
||
class TestRetention(_Base):
|
||
def test_noise_query_deleted_after_short_window(self):
|
||
self.make_task("query_limited_goods", age_days=10)
|
||
self.make_task("query_change_bind_time", age_days=10)
|
||
out = self.run(noise_days=7, business_days=90)
|
||
assert out["deleted"] == 2
|
||
assert out["deleted_noise"] == 2
|
||
assert self.remaining() == set()
|
||
|
||
def test_noise_query_kept_within_short_window(self):
|
||
self.make_task("query_limited_goods", age_days=3)
|
||
out = self.run(noise_days=7, business_days=90)
|
||
assert out["deleted"] == 0
|
||
assert self.count() == 1
|
||
|
||
def test_business_fact_kept_past_short_window(self):
|
||
"""业务事实不该被 7 天窗口带走 —— 这是分档的全部意义。"""
|
||
for task_type in (
|
||
"exchange_goods",
|
||
"create_gold_qr",
|
||
"confirm_bind",
|
||
"exchange_xpd_goods",
|
||
):
|
||
self.make_task(task_type, age_days=30)
|
||
out = self.run(noise_days=7, business_days=90)
|
||
assert out["deleted"] == 0
|
||
assert self.count() == 4
|
||
|
||
def test_business_fact_deleted_after_long_window(self):
|
||
self.make_task("exchange_goods", age_days=100)
|
||
out = self.run(noise_days=7, business_days=90)
|
||
assert out["deleted"] == 1
|
||
assert out["deleted_rest"] == 1
|
||
|
||
def test_state_derivation_queries_kept_longer(self):
|
||
"""工作台从这些查询的 result 反推账号当前状态,删早了显示会变「未查」。"""
|
||
for task_type in (
|
||
"query_xpd_bind_info",
|
||
"query_xpd_role",
|
||
"query_esports_game_name",
|
||
"query_game_name",
|
||
"prepare_esports_bind",
|
||
"get_esports_bind_qr",
|
||
):
|
||
self.make_task(task_type, age_days=30)
|
||
out = self.run(noise_days=7, business_days=90)
|
||
assert out["deleted"] == 0, f"状态派生依赖的任务被误删:{self.remaining()}"
|
||
|
||
def test_unknown_task_type_defaults_to_long_retention(self):
|
||
"""新增的任务类型默认长保留 —— 绝不能因为「没列进名单」而被删掉。"""
|
||
self.make_task("some_brand_new_task_type", age_days=30)
|
||
out = self.run(noise_days=7, business_days=90)
|
||
assert out["deleted"] == 0
|
||
assert self.count() == 1
|
||
|
||
# 但只要过了长窗口就该被清掉(否则未知类型会永久堆积)
|
||
self.make_task("another_new_type", age_days=200)
|
||
out = self.run(noise_days=7, business_days=90)
|
||
assert out["deleted"] == 1
|
||
|
||
def test_in_flight_tasks_never_deleted(self):
|
||
"""批次线程可能正拿着这些行,无论多老都不能删。"""
|
||
for status in ("planned", "pending", "running"):
|
||
self.make_task("query_limited_goods", age_days=999, status=status)
|
||
self.make_task("query_limited_goods", age_days=999, status="success")
|
||
out = self.run(noise_days=7, business_days=90)
|
||
assert out["deleted"] == 1
|
||
statuses = {t.status for t in self.session.query(DouyuTask).all()}
|
||
assert statuses == {"planned", "pending", "running"}
|
||
|
||
def test_config_windows_are_honoured(self):
|
||
self.make_task("query_limited_goods", age_days=2)
|
||
out = self.run(noise_days=1, business_days=90)
|
||
assert out["deleted"] == 1
|
||
assert out["noise_days"] == 1
|
||
|
||
def test_invalid_config_falls_back_to_defaults(self):
|
||
out = self.run(noise_days=0, business_days="abc")
|
||
assert out["noise_days"] == retention.DEFAULT_NOISE_DAYS
|
||
assert out["business_days"] == retention.DEFAULT_BUSINESS_DAYS
|
||
|
||
def test_result_payload_is_dropped_with_the_row(self):
|
||
"""删整行(而不是只清 result)—— 避免「有行但读不出状态」的隐蔽坏状态。"""
|
||
task = self.make_task(
|
||
"query_limited_goods",
|
||
age_days=30,
|
||
result={"limited_goods": [{"a": 1}] * 100},
|
||
)
|
||
task_id = task.id # 先取:run() 会 expunge_all,之后对象就 detached 了
|
||
self.run(noise_days=7, business_days=90)
|
||
assert self.session.get(DouyuTask, task_id) is None
|
||
|
||
def test_batch_budget_is_reported_not_overclaimed(self):
|
||
"""预算用尽只报「预算用光」,不谎称「仍有积压」。
|
||
|
||
预算在上一档用光时,下一档根本没轮到执行 —— 这时无法区分「没得删」和
|
||
「没轮到」,所以只给提示。
|
||
"""
|
||
for _ in range(5):
|
||
self.make_task("query_limited_goods", age_days=30)
|
||
out = self.run(noise_days=7, business_days=90, max_batches=1)
|
||
# batch_size=100(job 默认),5 条一轮删完(不足一批就 break),预算正好用光
|
||
assert out["batches"] == 1
|
||
assert out["deleted"] == 5
|
||
assert out["batch_budget_exhausted"] is True
|
||
|
||
def test_budget_not_exhausted_when_work_runs_out(self):
|
||
self.make_task("query_limited_goods", age_days=30)
|
||
out = self.run(noise_days=7, business_days=90, max_batches=50)
|
||
assert out["batch_budget_exhausted"] is False
|
||
|
||
|
||
class TestClassification(_Base):
|
||
def test_noise_list_has_no_duplicates(self):
|
||
assert len(retention.NOISE_TASK_TYPES) == len(set(retention.NOISE_TASK_TYPES))
|
||
|
||
def test_noise_list_has_no_business_types(self):
|
||
business = {
|
||
"exchange_goods",
|
||
"exchange_esports_goods",
|
||
"exchange_xpd_goods",
|
||
"pay_locked_order",
|
||
"lock_goods",
|
||
"create_gold_qr",
|
||
"donate_elite_gift",
|
||
"donate_esports_chicken_gift",
|
||
"donate_esports_firework_gift",
|
||
"confirm_bind",
|
||
"confirm_esports_bind",
|
||
"confirm_xpd_bind",
|
||
"create_elite_qr",
|
||
"create_esports_qr",
|
||
}
|
||
overlap = business & set(retention.NOISE_TASK_TYPES)
|
||
assert not overlap, f"业务类型被当成噪声了:{overlap}"
|
||
|
||
def test_noise_types_exist_in_supported_types(self):
|
||
"""噪声名单里的类型必须都是真实存在的任务类型(防拼写错误导致永不生效)。"""
|
||
from web.backend.services.douyu_service import SUPPORTED_DOUYU_TASK_TYPES
|
||
|
||
known = set(SUPPORTED_DOUYU_TASK_TYPES) | set(
|
||
retention.RETIRED_NOISE_TASK_TYPES
|
||
)
|
||
unknown = set(retention.NOISE_TASK_TYPES) - known
|
||
assert not unknown, f"噪声名单里有不存在的任务类型:{unknown}"
|