perf(db): 增加查询索引并提升 redo log 容量
This commit is contained in:
@@ -49,6 +49,7 @@ MYSQL_BIND_HOST=127.0.0.1
|
||||
MYSQL_HOST_PORT=3307
|
||||
# MySQL 性能观测;同机部署资源紧张时可先使用 256M。
|
||||
MYSQL_INNODB_BUFFER_POOL_SIZE=512M
|
||||
MYSQL_INNODB_REDO_LOG_CAPACITY=512M
|
||||
MYSQL_SLOW_QUERY_LOG=1
|
||||
MYSQL_LONG_QUERY_TIME=1
|
||||
|
||||
|
||||
@@ -156,7 +156,10 @@ cmd_deploy() {
|
||||
# 曾经这里是不带服务名的 `up -d --force-recreate`,会把 mysql 也一起重建 ——
|
||||
# 每次部署都有约 90 秒的 502(mysql 重启期间应用连不上库)。
|
||||
# 现在 mysql 只在它自己的配置真的变了时才动,代码部署不再影响数据库。
|
||||
$COMPOSE up -d douyu-login
|
||||
# Compose only recreates services whose configuration changed, so checking
|
||||
# mysql here applies redo-log/settings changes without restarting it every
|
||||
# deployment.
|
||||
$COMPOSE up -d mysql douyu-login
|
||||
|
||||
echo ""
|
||||
echo "等待服务启动..."
|
||||
|
||||
@@ -168,6 +168,8 @@ services:
|
||||
- --default-time-zone=+00:00
|
||||
# 业务表已超过 300MiB;可通过 .env 调整,默认给 MySQL 512MiB 缓冲池。
|
||||
- --innodb-buffer-pool-size=${MYSQL_INNODB_BUFFER_POOL_SIZE:-512M}
|
||||
# 100MiB redo 在高写入时出现过 checkpointer 落后告警,默认提升到 512MiB。
|
||||
- --innodb-redo-log-capacity=${MYSQL_INNODB_REDO_LOG_CAPACITY:-512M}
|
||||
# 默认记录 1 秒以上查询,便于验证查询优化;稳定后可设 MYSQL_SLOW_QUERY_LOG=0。
|
||||
- --slow-query-log=${MYSQL_SLOW_QUERY_LOG:-1}
|
||||
- --long-query-time=${MYSQL_LONG_QUERY_TIME:-1}
|
||||
|
||||
@@ -8,6 +8,7 @@ from sqlalchemy import func, or_
|
||||
|
||||
from web.backend.models import Account, DouyuTask, User
|
||||
from web.backend.routers.douyu import (
|
||||
_split_task_scope_query,
|
||||
_visible_tasks_query,
|
||||
list_tasks,
|
||||
)
|
||||
@@ -184,6 +185,20 @@ def _seed_tasks(db):
|
||||
return support, other, admin, no_scope_permission, tasks
|
||||
|
||||
|
||||
def test_legacy_task_scope_uses_per_type_index_ranges(db):
|
||||
branches = _split_task_scope_query(
|
||||
db.query(DouyuTask.id),
|
||||
"elite",
|
||||
)
|
||||
assert len(branches) == 1 + len(DOUYU_HANDBOOK_TASK_TYPES["elite"])
|
||||
legacy_sql = [
|
||||
str(branch.statement.compile(compile_kwargs={"literal_binds": True}))
|
||||
for branch in branches[1:]
|
||||
]
|
||||
assert all("task_type = " in statement for statement in legacy_sql)
|
||||
assert all(" IN (" not in statement for statement in legacy_sql)
|
||||
|
||||
|
||||
def test_support_task_list_matches_legacy_semantics(db):
|
||||
support, _, admin, no_scope_permission, tasks = _seed_tasks(db)
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ TESTS_DIR = Path(__file__).resolve().parent
|
||||
PROJECT_ROOT = TESTS_DIR.parent
|
||||
VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions"
|
||||
|
||||
HEAD_REVISION = "20261120_0101"
|
||||
HEAD_REVISION = "20261120_0102"
|
||||
|
||||
# 迁移新增、但不声明在模型里的复合查询索引。
|
||||
EXTRA_INDEXES = (
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
"""add account-name and audit-target query indexes
|
||||
|
||||
Production slow-query review found:
|
||||
- account lookups by username scanned the accounts table;
|
||||
- audit timeline lookups by target scanned audit_logs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20261120_0102"
|
||||
down_revision: str | None = "20261120_0101"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
INDEXES: tuple[tuple[str, str, tuple[str, ...]], ...] = (
|
||||
("ix_accounts_username_id", "accounts", ("username", "id")),
|
||||
("ix_audit_logs_target_id", "audit_logs", ("target", "id")),
|
||||
)
|
||||
|
||||
|
||||
def _indexes(table: str) -> set[str]:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(table):
|
||||
return set()
|
||||
return {
|
||||
str(index["name"])
|
||||
for index in inspector.get_indexes(table)
|
||||
if index.get("name")
|
||||
}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for name, table, columns in INDEXES:
|
||||
if name not in _indexes(table):
|
||||
op.create_index(name, table, list(columns), unique=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for name, table, _columns in reversed(INDEXES):
|
||||
if name in _indexes(table):
|
||||
op.drop_index(name, table_name=table)
|
||||
@@ -259,6 +259,9 @@ class Account(Base):
|
||||
archived_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
# Account-name lookup is used by imports, task creation and support
|
||||
# searches. Usernames are not unique in legacy data.
|
||||
Index("ix_accounts_username_id", "username", "id"),
|
||||
# 巡检按「最久没查的优先」选行
|
||||
Index("ix_accounts_ck_checked_at", "ck_checked_at"),
|
||||
# 重登按状态 + 退避时间选行
|
||||
@@ -1649,6 +1652,8 @@ class AuditLog(Base):
|
||||
success: Mapped[bool | None] = mapped_column(Boolean, nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow)
|
||||
|
||||
__table_args__ = (Index("ix_audit_logs_target_id", "target", "id"),)
|
||||
|
||||
|
||||
class AccountEvent(Base):
|
||||
"""账号事件流(append-only)。
|
||||
|
||||
@@ -291,6 +291,26 @@ def _visible_tasks_query(db: Session, current: User):
|
||||
raise HTTPException(status_code=403, detail="无权查看斗鱼任务")
|
||||
|
||||
|
||||
def _split_task_scope_query(query, handbook_scope: str | None) -> list:
|
||||
"""Build index-friendly branches for a workbench scope.
|
||||
|
||||
Legacy rows map to the requested workbench by task type. A single
|
||||
``task_type IN (...) ORDER BY id DESC`` query makes MySQL scan the broad
|
||||
handbook-scope index, so use one range per task type instead.
|
||||
"""
|
||||
if not handbook_scope:
|
||||
return [query]
|
||||
branches = [query.filter(DouyuTask.handbook_scope == handbook_scope)]
|
||||
branches.extend(
|
||||
query.filter(
|
||||
DouyuTask.handbook_scope == "legacy",
|
||||
DouyuTask.task_type == task_type,
|
||||
)
|
||||
for task_type in DOUYU_HANDBOOK_TASK_TYPES[handbook_scope]
|
||||
)
|
||||
return branches
|
||||
|
||||
|
||||
def _list_visible_tasks_for_support(
|
||||
db: Session,
|
||||
current: User,
|
||||
@@ -305,25 +325,13 @@ def _list_visible_tasks_for_support(
|
||||
(DouyuTask.account_assigned_to == current.id,),
|
||||
(DouyuTask.created_by == current.id,),
|
||||
]
|
||||
scope_filters: list[tuple[Any, ...]]
|
||||
if handbook_scope:
|
||||
scope_filters = [
|
||||
(DouyuTask.handbook_scope == handbook_scope,),
|
||||
(
|
||||
DouyuTask.handbook_scope == "legacy",
|
||||
DouyuTask.task_type.in_(DOUYU_HANDBOOK_TASK_TYPES[handbook_scope]),
|
||||
),
|
||||
]
|
||||
else:
|
||||
scope_filters = [()]
|
||||
|
||||
take = offset + limit
|
||||
task_ids: list[int] = []
|
||||
for owner_filter in owner_filters:
|
||||
for scope_filter in scope_filters:
|
||||
query = db.query(DouyuTask.id).filter(*owner_filter, *scope_filter)
|
||||
if batch_id:
|
||||
query = query.filter(DouyuTask.batch_id == batch_id)
|
||||
base_query = db.query(DouyuTask.id).filter(*owner_filter)
|
||||
if batch_id:
|
||||
base_query = base_query.filter(DouyuTask.batch_id == batch_id)
|
||||
for query in _split_task_scope_query(base_query, handbook_scope):
|
||||
rows = query.order_by(DouyuTask.id.desc()).limit(take).all()
|
||||
task_ids.extend(int(row[0]) for row in rows)
|
||||
|
||||
@@ -359,23 +367,12 @@ def _list_visible_tasks_for_admin(
|
||||
offset: int = 0,
|
||||
) -> list[DouyuTask]:
|
||||
"""管理员任务列表按作用域拆索引分支,避免 OR + JOIN 触发大排序。"""
|
||||
if handbook_scope:
|
||||
scope_filters: list[tuple[Any, ...]] = [
|
||||
(DouyuTask.handbook_scope == handbook_scope,),
|
||||
(
|
||||
DouyuTask.handbook_scope == "legacy",
|
||||
DouyuTask.task_type.in_(DOUYU_HANDBOOK_TASK_TYPES[handbook_scope]),
|
||||
),
|
||||
]
|
||||
else:
|
||||
scope_filters = [()]
|
||||
|
||||
take = offset + limit
|
||||
task_ids: list[int] = []
|
||||
for scope_filter in scope_filters:
|
||||
query = db.query(DouyuTask.id).filter(*scope_filter)
|
||||
if batch_id:
|
||||
query = query.filter(DouyuTask.batch_id == batch_id)
|
||||
base_query = db.query(DouyuTask.id)
|
||||
if batch_id:
|
||||
base_query = base_query.filter(DouyuTask.batch_id == batch_id)
|
||||
for query in _split_task_scope_query(base_query, handbook_scope):
|
||||
rows = query.order_by(DouyuTask.id.desc()).limit(take).all()
|
||||
task_ids.extend(int(row[0]) for row in rows)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user