perf(db): 增加查询索引并提升 redo log 容量

This commit is contained in:
yml
2026-11-20 22:26:10 +08:00
parent 409e22be90
commit 2eff8e1124
8 changed files with 100 additions and 33 deletions
+1
View File
@@ -49,6 +49,7 @@ MYSQL_BIND_HOST=127.0.0.1
MYSQL_HOST_PORT=3307
# MySQL 性能观测;同机部署资源紧张时可先使用 256M。
MYSQL_INNODB_BUFFER_POOL_SIZE=512M
MYSQL_INNODB_REDO_LOG_CAPACITY=512M
MYSQL_SLOW_QUERY_LOG=1
MYSQL_LONG_QUERY_TIME=1
+4 -1
View File
@@ -156,7 +156,10 @@ cmd_deploy() {
# 曾经这里是不带服务名的 `up -d --force-recreate`,会把 mysql 也一起重建 ——
# 每次部署都有约 90 秒的 502(mysql 重启期间应用连不上库)。
# 现在 mysql 只在它自己的配置真的变了时才动,代码部署不再影响数据库。
$COMPOSE up -d douyu-login
# Compose only recreates services whose configuration changed, so checking
# mysql here applies redo-log/settings changes without restarting it every
# deployment.
$COMPOSE up -d mysql douyu-login
echo ""
echo "等待服务启动..."
+2
View File
@@ -168,6 +168,8 @@ services:
- --default-time-zone=+00:00
# 业务表已超过 300MiB;可通过 .env 调整,默认给 MySQL 512MiB 缓冲池。
- --innodb-buffer-pool-size=${MYSQL_INNODB_BUFFER_POOL_SIZE:-512M}
# 100MiB redo 在高写入时出现过 checkpointer 落后告警,默认提升到 512MiB。
- --innodb-redo-log-capacity=${MYSQL_INNODB_REDO_LOG_CAPACITY:-512M}
# 默认记录 1 秒以上查询,便于验证查询优化;稳定后可设 MYSQL_SLOW_QUERY_LOG=0。
- --slow-query-log=${MYSQL_SLOW_QUERY_LOG:-1}
- --long-query-time=${MYSQL_LONG_QUERY_TIME:-1}
+15
View File
@@ -8,6 +8,7 @@ from sqlalchemy import func, or_
from web.backend.models import Account, DouyuTask, User
from web.backend.routers.douyu import (
_split_task_scope_query,
_visible_tasks_query,
list_tasks,
)
@@ -184,6 +185,20 @@ def _seed_tasks(db):
return support, other, admin, no_scope_permission, tasks
def test_legacy_task_scope_uses_per_type_index_ranges(db):
branches = _split_task_scope_query(
db.query(DouyuTask.id),
"elite",
)
assert len(branches) == 1 + len(DOUYU_HANDBOOK_TASK_TYPES["elite"])
legacy_sql = [
str(branch.statement.compile(compile_kwargs={"literal_binds": True}))
for branch in branches[1:]
]
assert all("task_type = " in statement for statement in legacy_sql)
assert all(" IN (" not in statement for statement in legacy_sql)
def test_support_task_list_matches_legacy_semantics(db):
support, _, admin, no_scope_permission, tasks = _seed_tasks(db)
+1 -1
View File
@@ -40,7 +40,7 @@ TESTS_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = TESTS_DIR.parent
VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions"
HEAD_REVISION = "20261120_0101"
HEAD_REVISION = "20261120_0102"
# 迁移新增、但不声明在模型里的复合查询索引。
EXTRA_INDEXES = (
@@ -0,0 +1,44 @@
"""add account-name and audit-target query indexes
Production slow-query review found:
- account lookups by username scanned the accounts table;
- audit timeline lookups by target scanned audit_logs.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision: str = "20261120_0102"
down_revision: str | None = "20261120_0101"
branch_labels: str | None = None
depends_on: str | None = None
INDEXES: tuple[tuple[str, str, tuple[str, ...]], ...] = (
("ix_accounts_username_id", "accounts", ("username", "id")),
("ix_audit_logs_target_id", "audit_logs", ("target", "id")),
)
def _indexes(table: str) -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(table):
return set()
return {
str(index["name"])
for index in inspector.get_indexes(table)
if index.get("name")
}
def upgrade() -> None:
for name, table, columns in INDEXES:
if name not in _indexes(table):
op.create_index(name, table, list(columns), unique=False)
def downgrade() -> None:
for name, table, _columns in reversed(INDEXES):
if name in _indexes(table):
op.drop_index(name, table_name=table)
+5
View File
@@ -259,6 +259,9 @@ class Account(Base):
archived_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True)
__table_args__ = (
# Account-name lookup is used by imports, task creation and support
# searches. Usernames are not unique in legacy data.
Index("ix_accounts_username_id", "username", "id"),
# 巡检按「最久没查的优先」选行
Index("ix_accounts_ck_checked_at", "ck_checked_at"),
# 重登按状态 + 退避时间选行
@@ -1649,6 +1652,8 @@ class AuditLog(Base):
success: Mapped[bool | None] = mapped_column(Boolean, nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow)
__table_args__ = (Index("ix_audit_logs_target_id", "target", "id"),)
class AccountEvent(Base):
"""账号事件流(append-only)。
+28 -31
View File
@@ -291,6 +291,26 @@ def _visible_tasks_query(db: Session, current: User):
raise HTTPException(status_code=403, detail="无权查看斗鱼任务")
def _split_task_scope_query(query, handbook_scope: str | None) -> list:
"""Build index-friendly branches for a workbench scope.
Legacy rows map to the requested workbench by task type. A single
``task_type IN (...) ORDER BY id DESC`` query makes MySQL scan the broad
handbook-scope index, so use one range per task type instead.
"""
if not handbook_scope:
return [query]
branches = [query.filter(DouyuTask.handbook_scope == handbook_scope)]
branches.extend(
query.filter(
DouyuTask.handbook_scope == "legacy",
DouyuTask.task_type == task_type,
)
for task_type in DOUYU_HANDBOOK_TASK_TYPES[handbook_scope]
)
return branches
def _list_visible_tasks_for_support(
db: Session,
current: User,
@@ -305,25 +325,13 @@ def _list_visible_tasks_for_support(
(DouyuTask.account_assigned_to == current.id,),
(DouyuTask.created_by == current.id,),
]
scope_filters: list[tuple[Any, ...]]
if handbook_scope:
scope_filters = [
(DouyuTask.handbook_scope == handbook_scope,),
(
DouyuTask.handbook_scope == "legacy",
DouyuTask.task_type.in_(DOUYU_HANDBOOK_TASK_TYPES[handbook_scope]),
),
]
else:
scope_filters = [()]
take = offset + limit
task_ids: list[int] = []
for owner_filter in owner_filters:
for scope_filter in scope_filters:
query = db.query(DouyuTask.id).filter(*owner_filter, *scope_filter)
if batch_id:
query = query.filter(DouyuTask.batch_id == batch_id)
base_query = db.query(DouyuTask.id).filter(*owner_filter)
if batch_id:
base_query = base_query.filter(DouyuTask.batch_id == batch_id)
for query in _split_task_scope_query(base_query, handbook_scope):
rows = query.order_by(DouyuTask.id.desc()).limit(take).all()
task_ids.extend(int(row[0]) for row in rows)
@@ -359,23 +367,12 @@ def _list_visible_tasks_for_admin(
offset: int = 0,
) -> list[DouyuTask]:
"""管理员任务列表按作用域拆索引分支,避免 OR + JOIN 触发大排序。"""
if handbook_scope:
scope_filters: list[tuple[Any, ...]] = [
(DouyuTask.handbook_scope == handbook_scope,),
(
DouyuTask.handbook_scope == "legacy",
DouyuTask.task_type.in_(DOUYU_HANDBOOK_TASK_TYPES[handbook_scope]),
),
]
else:
scope_filters = [()]
take = offset + limit
task_ids: list[int] = []
for scope_filter in scope_filters:
query = db.query(DouyuTask.id).filter(*scope_filter)
if batch_id:
query = query.filter(DouyuTask.batch_id == batch_id)
base_query = db.query(DouyuTask.id)
if batch_id:
base_query = base_query.filter(DouyuTask.batch_id == batch_id)
for query in _split_task_scope_query(base_query, handbook_scope):
rows = query.order_by(DouyuTask.id.desc()).limit(take).all()
task_ids.extend(int(row[0]) for row in rows)