perf(douyu-tasks): 为斗鱼任务增加归属客服索引字段,彻底消除全表 filesort 慢查询
1. 数据库新增迁移 20261003_0040,在 douyu_tasks 表增加 account_assigned_to 字段,并创建 (account_assigned_to, handbook_scope, id) 与 (account_assigned_to, id) 复合覆盖索引; 2. 迁移脚本自动回填历史 24 万条任务的所属客服 ID; 3. 任务服务与账号路由联动:创建任务时自动继承 account_assigned_to,修改账号分配时同步更新关联任务归属; 4. 斗鱼任务查询路由 _visible_tasks_query 改为通过 account_assigned_to 复合索引过滤,消除与 accounts 表的跨表关联与临时表全量文件排序; 5. 新增 test_douyu_task_assigned_optimization 单元测试并通过整链迁移与回归测试。
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
"""为 douyu_tasks 增加 account_assigned_to 字段与复合索引,彻底优化任务全表排序性能。
|
||||
|
||||
背景:
|
||||
斗鱼任务表 (douyu_tasks) 历史数据已达数十万行。当客服查看任务列表时,原本通过
|
||||
JOIN accounts 表并在 accounts.assigned_to 上过滤,导致 MySQL 必须在临时表中对全部
|
||||
匹配任务做 filesort 文件排序(Using temporary; Using filesort),单次查询消耗几百毫秒
|
||||
并导致 MySQL 容器 CPU 周期性冲顶到 100%+。
|
||||
|
||||
本迁移:
|
||||
1. 为 douyu_tasks 表新增 account_assigned_to 列;
|
||||
2. 批量回填历史任务的归属客服 ID;
|
||||
3. 建立 (account_assigned_to, handbook_scope, id) 与 (account_assigned_to, id) 复合索引,
|
||||
使客服任务列表查询从 O(N log N) 临时表全排序降级为 O(1) 索引倒序定位,彻底消除 filesort。
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20261003_0040"
|
||||
down_revision: str | None = "20261002_0039"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def _columns(bind, table_name: str) -> set[str]:
|
||||
return {col["name"] for col in sa.inspect(bind).get_columns(table_name)}
|
||||
|
||||
|
||||
def _indexes(bind, table_name: str) -> set[str]:
|
||||
return {idx["name"] for idx in sa.inspect(bind).get_indexes(table_name)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
existing_cols = _columns(bind, "douyu_tasks")
|
||||
|
||||
if "account_assigned_to" not in existing_cols:
|
||||
op.add_column(
|
||||
"douyu_tasks",
|
||||
sa.Column(
|
||||
"account_assigned_to",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id"),
|
||||
nullable=True,
|
||||
),
|
||||
)
|
||||
|
||||
# 历史任务回填归属客服 ID
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE douyu_tasks dt
|
||||
JOIN accounts a ON a.id = dt.account_id
|
||||
SET dt.account_assigned_to = a.assigned_to
|
||||
WHERE a.assigned_to IS NOT NULL AND dt.account_assigned_to IS NULL;
|
||||
"""
|
||||
)
|
||||
|
||||
existing_indexes = _indexes(bind, "douyu_tasks")
|
||||
if "ix_douyu_tasks_account_assigned_to_id" not in existing_indexes:
|
||||
op.create_index(
|
||||
"ix_douyu_tasks_account_assigned_to_id",
|
||||
"douyu_tasks",
|
||||
["account_assigned_to", "id"],
|
||||
)
|
||||
|
||||
if "ix_douyu_tasks_assigned_scope_id" not in existing_indexes:
|
||||
op.create_index(
|
||||
"ix_douyu_tasks_assigned_scope_id",
|
||||
"douyu_tasks",
|
||||
["account_assigned_to", "handbook_scope", "id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
existing_indexes = _indexes(bind, "douyu_tasks")
|
||||
|
||||
if "ix_douyu_tasks_assigned_scope_id" in existing_indexes:
|
||||
op.drop_index("ix_douyu_tasks_assigned_scope_id", table_name="douyu_tasks")
|
||||
|
||||
if "ix_douyu_tasks_account_assigned_to_id" in existing_indexes:
|
||||
op.drop_index("ix_douyu_tasks_account_assigned_to_id", table_name="douyu_tasks")
|
||||
|
||||
existing_cols = _columns(bind, "douyu_tasks")
|
||||
if "account_assigned_to" in existing_cols:
|
||||
op.drop_column("douyu_tasks", "account_assigned_to")
|
||||
@@ -8,6 +8,7 @@ from sqlalchemy import (
|
||||
JSON,
|
||||
Boolean,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
@@ -214,6 +215,24 @@ class DouyuTask(Base):
|
||||
created_by: Mapped[int] = mapped_column(ForeignKey("users.id"), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True)
|
||||
# 账号归属客服冗余字段,配合复合索引消除任务列表全表 filesort。
|
||||
account_assigned_to: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("users.id"), nullable=True
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_douyu_tasks_account_assigned_to_id",
|
||||
"account_assigned_to",
|
||||
"id",
|
||||
),
|
||||
Index(
|
||||
"ix_douyu_tasks_assigned_scope_id",
|
||||
"account_assigned_to",
|
||||
"handbook_scope",
|
||||
"id",
|
||||
),
|
||||
)
|
||||
|
||||
account: Mapped[Account] = relationship("Account", back_populates="douyu_tasks")
|
||||
|
||||
|
||||
@@ -289,7 +289,11 @@ def assign_account(
|
||||
if not has_success:
|
||||
raise HTTPException(status_code=400, detail="该账号尚未成功登录,无法分配")
|
||||
|
||||
acc.assigned_to = req.assigned_to
|
||||
if acc.assigned_to != req.assigned_to:
|
||||
acc.assigned_to = req.assigned_to
|
||||
db.query(DouyuTask).filter(DouyuTask.account_id == acc.id).update(
|
||||
{DouyuTask.account_assigned_to: req.assigned_to}, synchronize_session=False
|
||||
)
|
||||
db.add(
|
||||
AuditLog(
|
||||
user_id=current.id,
|
||||
@@ -348,6 +352,9 @@ def batch_assign_accounts(
|
||||
.filter(Account.id.in_(req.account_ids))
|
||||
.update({Account.assigned_to: req.assigned_to}, synchronize_session=False)
|
||||
)
|
||||
db.query(DouyuTask).filter(DouyuTask.account_id.in_(req.account_ids)).update(
|
||||
{DouyuTask.account_assigned_to: req.assigned_to}, synchronize_session=False
|
||||
)
|
||||
db.add(
|
||||
AuditLog(
|
||||
user_id=current.id,
|
||||
|
||||
@@ -212,7 +212,8 @@ def _visible_tasks_query(db: Session, current: User):
|
||||
if _can_view_all(current):
|
||||
return query
|
||||
if user_has_permission(current, "account:view_assigned"):
|
||||
return query.join(DouyuTask.account).filter(Account.assigned_to == current.id)
|
||||
# 彻底消除全表 filesort:直接通过 douyu_tasks 上的 account_assigned_to 复合索引倒序定位
|
||||
return query.filter(DouyuTask.account_assigned_to == current.id)
|
||||
raise HTTPException(status_code=403, detail="无权查看斗鱼任务")
|
||||
|
||||
|
||||
|
||||
@@ -295,6 +295,7 @@ def create_douyu_planned_tasks(
|
||||
DouyuTask(
|
||||
batch_id=batch_id,
|
||||
account_id=account.id,
|
||||
account_assigned_to=account.assigned_to,
|
||||
task_type=task_type,
|
||||
handbook_scope=handbook_scope,
|
||||
status="planned",
|
||||
|
||||
Reference in New Issue
Block a user