perf(douyu): 优化任务列表查询 SQL

This commit is contained in:
2026-11-20 19:38:54 +08:00
parent 4d8f28666c
commit 2186552b96
2 changed files with 447 additions and 25 deletions
+300
View File
@@ -0,0 +1,300 @@
"""任务列表 SQL 拆分后与旧查询语义保持一致的回归测试。"""
from __future__ import annotations
import pytest
from fastapi import HTTPException
from sqlalchemy import func, or_
from web.backend.models import Account, DouyuTask, User
from web.backend.routers.douyu import (
_visible_tasks_query,
list_tasks,
)
from web.backend.services.douyu_service import DOUYU_HANDBOOK_TASK_TYPES
def _legacy_page(
db,
user: User,
*,
batch_id: str | None,
handbook_scope: str | None,
page: int | None,
page_size: int,
):
"""复刻改造前的 ORM 查询,作为等价性测试的基准。"""
query = _visible_tasks_query(db, user)
if batch_id:
query = query.filter(DouyuTask.batch_id == batch_id)
if handbook_scope:
query = query.filter(
or_(
DouyuTask.handbook_scope == handbook_scope,
(DouyuTask.handbook_scope == "legacy")
& DouyuTask.task_type.in_(DOUYU_HANDBOOK_TASK_TYPES[handbook_scope]),
)
)
total = None
if page is not None:
total = (
query.enable_eagerloads(False)
.order_by(None)
.with_entities(func.count(DouyuTask.id))
.scalar()
or 0
)
query = query.order_by(DouyuTask.id.desc())
if page is not None:
query = query.offset((page - 1) * page_size).limit(page_size)
else:
query = query.limit(page_size)
return [task.id for task in query.all()], total
def _new_page(result):
if isinstance(result, dict):
return [item.id for item in result["items"]], result["total"]
return [item.id for item in result], None
def _seed_tasks(db):
support = User(
username="task-list-support",
password_hash="hash",
role="support",
is_active=True,
)
other = User(
username="task-list-other",
password_hash="hash",
role="support",
is_active=True,
)
admin = User(
username="task-list-admin",
password_hash="hash",
role="super_admin",
is_active=True,
)
no_scope_permission = User(
username="task-list-no-scope",
password_hash="hash",
role="support",
custom_permissions=["douyu:task"],
is_active=True,
)
db.add_all([support, other, admin, no_scope_permission])
db.flush()
support_account = Account(
username="task-list-account-support",
password="password",
email="support@example.com",
email_password="mail-password",
cookie="acf_uid=1",
assigned_to=support.id,
)
other_account = Account(
username="task-list-account-other",
password="password",
email="other@example.com",
email_password="mail-password",
cookie="acf_uid=2",
assigned_to=other.id,
)
db.add_all([support_account, other_account])
db.flush()
tasks = {
"assigned_elite": DouyuTask(
batch_id="batch-main",
account_id=support_account.id,
account_assigned_to=support.id,
task_type="query_points",
handbook_scope="elite",
status="success",
created_by=other.id,
result={"kind": "assigned"},
),
"created_elite": DouyuTask(
batch_id="batch-main",
account_id=other_account.id,
account_assigned_to=other.id,
task_type="query_account_overview",
handbook_scope="elite",
status="success",
created_by=support.id,
result={"kind": "created"},
),
"both_legacy_elite": DouyuTask(
batch_id="batch-main",
account_id=support_account.id,
account_assigned_to=support.id,
task_type="query_limited_goods",
handbook_scope="legacy",
status="success",
created_by=support.id,
result={"kind": "legacy-elite"},
),
"legacy_not_visible": DouyuTask(
batch_id="batch-main",
account_id=support_account.id,
account_assigned_to=support.id,
task_type="not_a_supported_task_type",
handbook_scope="legacy",
status="success",
created_by=support.id,
result={"kind": "legacy-hidden"},
),
"esports": DouyuTask(
batch_id="batch-main",
account_id=support_account.id,
account_assigned_to=support.id,
task_type="query_esports_points",
handbook_scope="esports",
status="success",
created_by=support.id,
result={"kind": "esports"},
),
"peace": DouyuTask(
batch_id="batch-main",
account_id=support_account.id,
account_assigned_to=support.id,
task_type="query_xpd_balance",
handbook_scope="peace",
status="success",
created_by=support.id,
result={"kind": "peace"},
),
"other_user": DouyuTask(
batch_id="batch-other",
account_id=other_account.id,
account_assigned_to=other.id,
task_type="query_points",
handbook_scope="elite",
status="success",
created_by=other.id,
result={"kind": "other-user"},
),
}
db.add_all(tasks.values())
db.commit()
return support, other, admin, no_scope_permission, tasks
def test_support_task_list_matches_legacy_semantics(db):
support, _, admin, no_scope_permission, tasks = _seed_tasks(db)
cases = [
(None, None, None, 100),
(None, None, 1, 2),
(None, "elite", None, 100),
(None, "esports", None, 100),
(None, "peace", None, 100),
("batch-main", None, None, 100),
("batch-main", "elite", None, 100),
("batch-main", "elite", 1, 2),
("batch-main", "elite", 2, 2),
("batch-main", "elite", 3, 2),
("batch-main", "elite", 1, 500),
("batch-main", "esports", 1, 2),
("batch-other", None, None, 100),
]
for batch_id, scope, page, page_size in cases:
expected_ids, expected_total = _legacy_page(
db,
support,
batch_id=batch_id,
handbook_scope=scope,
page=page,
page_size=page_size,
)
actual = list_tasks(
batch_id=batch_id,
handbook_scope=scope,
include_detail=False,
page=page,
page_size=page_size,
db=db,
current=support,
)
actual_ids, actual_total = _new_page(actual)
assert actual_ids == expected_ids, (batch_id, scope, page, page_size)
assert actual_total == expected_total, (batch_id, scope, page, page_size)
# 账号归属和创建人同时命中时不能重复。
both_id = tasks["both_legacy_elite"].id
elite = list_tasks(
batch_id="batch-main",
handbook_scope="elite",
include_detail=False,
page=None,
page_size=100,
db=db,
current=support,
)
assert [item.id for item in elite].count(both_id) == 1
assert all(item.account_username for item in elite)
# 管理员路径保持全量可见范围。
expected_ids, expected_total = _legacy_page(
db,
admin,
batch_id=None,
handbook_scope="elite",
page=1,
page_size=100,
)
admin_page = list_tasks(
batch_id=None,
handbook_scope="elite",
include_detail=False,
page=1,
page_size=100,
db=db,
current=admin,
)
assert _new_page(admin_page) == (expected_ids, expected_total)
# 没有 account:view_assigned 的普通任务用户仍应被拒绝。
with pytest.raises(HTTPException) as exc_info:
list_tasks(
batch_id=None,
handbook_scope=None,
include_detail=False,
page=None,
page_size=100,
db=db,
current=no_scope_permission,
)
assert exc_info.value.status_code == 403
def test_empty_support_task_list_stays_empty(db):
support = User(
username="empty-task-support",
password_hash="hash",
role="support",
is_active=True,
)
db.add(support)
db.commit()
rows = list_tasks(
batch_id=None,
handbook_scope="elite",
include_detail=False,
page=1,
page_size=100,
db=db,
current=support,
)
assert rows == {
"items": [],
"total": 0,
"page": 1,
"page_size": 100,
}
+147 -25
View File
@@ -291,6 +291,103 @@ def _visible_tasks_query(db: Session, current: User):
raise HTTPException(status_code=403, detail="无权查看斗鱼任务")
def _list_visible_tasks_for_support(
db: Session,
current: User,
*,
batch_id: str | None,
handbook_scope: str | None,
limit: int,
offset: int = 0,
) -> list[DouyuTask]:
"""按归属/创建人和工作台范围拆成索引分支,合并后加载完整任务。"""
owner_filters: list[tuple[Any, ...]] = [
(DouyuTask.account_assigned_to == current.id,),
(DouyuTask.created_by == current.id,),
]
scope_filters: list[tuple[Any, ...]]
if handbook_scope:
scope_filters = [
(DouyuTask.handbook_scope == handbook_scope,),
(
DouyuTask.handbook_scope == "legacy",
DouyuTask.task_type.in_(DOUYU_HANDBOOK_TASK_TYPES[handbook_scope]),
),
]
else:
scope_filters = [()]
take = offset + limit
task_ids: list[int] = []
for owner_filter in owner_filters:
for scope_filter in scope_filters:
query = db.query(DouyuTask.id).filter(*owner_filter, *scope_filter)
if batch_id:
query = query.filter(DouyuTask.batch_id == batch_id)
rows = query.order_by(DouyuTask.id.desc()).limit(take).all()
task_ids.extend(int(row[0]) for row in rows)
page_ids = sorted(set(task_ids), reverse=True)[offset : offset + limit]
if not page_ids:
return []
account_loader = joinedload(DouyuTask.account).load_only(
Account.id,
Account.username,
Account.uid,
Account.nickname,
Account.assigned_to,
)
tasks_by_id = {
task.id: task
for task in (
db.query(DouyuTask)
.options(account_loader)
.filter(DouyuTask.id.in_(page_ids))
.all()
)
}
return [tasks_by_id[task_id] for task_id in page_ids if task_id in tasks_by_id]
def _count_visible_tasks_for_support(
db: Session,
current: User,
*,
batch_id: str | None,
handbook_scope: str | None,
) -> int:
"""用索引分支精确计算归属/创建人并集的任务总数。"""
if handbook_scope:
scope_filters: list[tuple[Any, ...]] = [
(DouyuTask.handbook_scope == handbook_scope,),
(
DouyuTask.handbook_scope == "legacy",
DouyuTask.task_type.in_(DOUYU_HANDBOOK_TASK_TYPES[handbook_scope]),
),
]
else:
scope_filters = [()]
def count_owner(
owner_filter: tuple[Any, ...], scope_filter: tuple[Any, ...]
) -> int:
query = db.query(func.count(DouyuTask.id)).filter(*owner_filter, *scope_filter)
if batch_id:
query = query.filter(DouyuTask.batch_id == batch_id)
return int(query.scalar() or 0)
assigned_filter = (DouyuTask.account_assigned_to == current.id,)
created_filter = (DouyuTask.created_by == current.id,)
both_filter = assigned_filter + created_filter
total = 0
for scope_filter in scope_filters:
total += count_owner(assigned_filter, scope_filter)
total += count_owner(created_filter, scope_filter)
total -= count_owner(both_filter, scope_filter)
return total
def _require_task_account_access(
db: Session,
current: User,
@@ -996,33 +1093,58 @@ def list_tasks(
current: User = Depends(require_permission("douyu:task")),
):
"""查看斗鱼任务记录,默认只返回最近 100 条。"""
query = _visible_tasks_query(db, current)
if batch_id:
query = query.filter(DouyuTask.batch_id == batch_id)
if handbook_scope:
# 旧任务没有归属字段,按历史任务类型继续展示,但绝不会触发自动二维码弹窗。
query = query.filter(
or_(
DouyuTask.handbook_scope == handbook_scope,
(DouyuTask.handbook_scope == "legacy")
& DouyuTask.task_type.in_(DOUYU_HANDBOOK_TASK_TYPES[handbook_scope]),
)
)
total = None
if page is not None:
total = (
query.enable_eagerloads(False)
.order_by(None)
.with_entities(func.count(DouyuTask.id))
.scalar()
or 0
)
query = query.order_by(DouyuTask.id.desc())
if page is not None:
query = query.offset((page - 1) * page_size).limit(page_size)
offset = (page - 1) * page_size if page is not None else 0
if _can_view_all(current):
# 管理员保留原有全量查询路径,避免改变管理端的可见范围。
query = _visible_tasks_query(db, current)
if batch_id:
query = query.filter(DouyuTask.batch_id == batch_id)
if handbook_scope:
query = query.filter(
or_(
DouyuTask.handbook_scope == handbook_scope,
(DouyuTask.handbook_scope == "legacy")
& DouyuTask.task_type.in_(
DOUYU_HANDBOOK_TASK_TYPES[handbook_scope]
),
)
)
if page is not None:
total = (
query.enable_eagerloads(False)
.order_by(None)
.with_entities(func.count(DouyuTask.id))
.scalar()
or 0
)
query = query.order_by(DouyuTask.id.desc())
if page is not None:
query = query.offset(offset).limit(page_size)
else:
query = query.limit(page_size)
tasks = query.all()
else:
query = query.limit(page_size)
result = [_task_out(task, include_detail=include_detail) for task in query.all()]
if not user_has_permission(current, "account:view_assigned"):
raise HTTPException(status_code=403, detail="无权查看斗鱼任务")
tasks = _list_visible_tasks_for_support(
db,
current,
batch_id=batch_id,
handbook_scope=handbook_scope,
limit=page_size,
offset=offset,
)
if page is not None:
total = _count_visible_tasks_for_support(
db,
current,
batch_id=batch_id,
handbook_scope=handbook_scope,
)
result = [_task_out(task, include_detail=include_detail) for task in tasks]
if page is not None:
return {
"items": result,