From 2eff8e1124ef80a891efdaa0e923a2c27c5a8776 Mon Sep 17 00:00:00 2001 From: yml Date: Fri, 20 Nov 2026 22:26:10 +0800 Subject: [PATCH] =?UTF-8?q?perf(db):=20=E5=A2=9E=E5=8A=A0=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E7=B4=A2=E5=BC=95=E5=B9=B6=E6=8F=90=E5=8D=87=20redo?= =?UTF-8?q?=20log=20=E5=AE=B9=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 1 + deploy.sh | 5 +- docker-compose.yml | 2 + tests/test_douyu_task_list_query.py | 15 +++++ tests/test_migrations.py | 2 +- .../versions/20261120_0102_query_indexes.py | 44 ++++++++++++++ web/backend/models.py | 5 ++ web/backend/routers/douyu.py | 59 +++++++++---------- 8 files changed, 100 insertions(+), 33 deletions(-) create mode 100644 web/backend/migrations/versions/20261120_0102_query_indexes.py diff --git a/.env.example b/.env.example index 9f38f9c..0ab473a 100644 --- a/.env.example +++ b/.env.example @@ -49,6 +49,7 @@ MYSQL_BIND_HOST=127.0.0.1 MYSQL_HOST_PORT=3307 # MySQL 性能观测;同机部署资源紧张时可先使用 256M。 MYSQL_INNODB_BUFFER_POOL_SIZE=512M +MYSQL_INNODB_REDO_LOG_CAPACITY=512M MYSQL_SLOW_QUERY_LOG=1 MYSQL_LONG_QUERY_TIME=1 diff --git a/deploy.sh b/deploy.sh index 79ecd08..92a8dd4 100755 --- a/deploy.sh +++ b/deploy.sh @@ -156,7 +156,10 @@ cmd_deploy() { # 曾经这里是不带服务名的 `up -d --force-recreate`,会把 mysql 也一起重建 —— # 每次部署都有约 90 秒的 502(mysql 重启期间应用连不上库)。 # 现在 mysql 只在它自己的配置真的变了时才动,代码部署不再影响数据库。 - $COMPOSE up -d douyu-login + # Compose only recreates services whose configuration changed, so checking + # mysql here applies redo-log/settings changes without restarting it every + # deployment. + $COMPOSE up -d mysql douyu-login echo "" echo "等待服务启动..." diff --git a/docker-compose.yml b/docker-compose.yml index 31efdea..25c3f19 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -168,6 +168,8 @@ services: - --default-time-zone=+00:00 # 业务表已超过 300MiB;可通过 .env 调整,默认给 MySQL 512MiB 缓冲池。 - --innodb-buffer-pool-size=${MYSQL_INNODB_BUFFER_POOL_SIZE:-512M} + # 100MiB redo 在高写入时出现过 checkpointer 落后告警,默认提升到 512MiB。 + - --innodb-redo-log-capacity=${MYSQL_INNODB_REDO_LOG_CAPACITY:-512M} # 默认记录 1 秒以上查询,便于验证查询优化;稳定后可设 MYSQL_SLOW_QUERY_LOG=0。 - --slow-query-log=${MYSQL_SLOW_QUERY_LOG:-1} - --long-query-time=${MYSQL_LONG_QUERY_TIME:-1} diff --git a/tests/test_douyu_task_list_query.py b/tests/test_douyu_task_list_query.py index ed8acf3..f2d3b01 100644 --- a/tests/test_douyu_task_list_query.py +++ b/tests/test_douyu_task_list_query.py @@ -8,6 +8,7 @@ from sqlalchemy import func, or_ from web.backend.models import Account, DouyuTask, User from web.backend.routers.douyu import ( + _split_task_scope_query, _visible_tasks_query, list_tasks, ) @@ -184,6 +185,20 @@ def _seed_tasks(db): return support, other, admin, no_scope_permission, tasks +def test_legacy_task_scope_uses_per_type_index_ranges(db): + branches = _split_task_scope_query( + db.query(DouyuTask.id), + "elite", + ) + assert len(branches) == 1 + len(DOUYU_HANDBOOK_TASK_TYPES["elite"]) + legacy_sql = [ + str(branch.statement.compile(compile_kwargs={"literal_binds": True})) + for branch in branches[1:] + ] + assert all("task_type = " in statement for statement in legacy_sql) + assert all(" IN (" not in statement for statement in legacy_sql) + + def test_support_task_list_matches_legacy_semantics(db): support, _, admin, no_scope_permission, tasks = _seed_tasks(db) diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 5a196a9..a65d72d 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -40,7 +40,7 @@ TESTS_DIR = Path(__file__).resolve().parent PROJECT_ROOT = TESTS_DIR.parent VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions" -HEAD_REVISION = "20261120_0101" +HEAD_REVISION = "20261120_0102" # 迁移新增、但不声明在模型里的复合查询索引。 EXTRA_INDEXES = ( diff --git a/web/backend/migrations/versions/20261120_0102_query_indexes.py b/web/backend/migrations/versions/20261120_0102_query_indexes.py new file mode 100644 index 0000000..c502dc5 --- /dev/null +++ b/web/backend/migrations/versions/20261120_0102_query_indexes.py @@ -0,0 +1,44 @@ +"""add account-name and audit-target query indexes + +Production slow-query review found: +- account lookups by username scanned the accounts table; +- audit timeline lookups by target scanned audit_logs. +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision: str = "20261120_0102" +down_revision: str | None = "20261120_0101" +branch_labels: str | None = None +depends_on: str | None = None + +INDEXES: tuple[tuple[str, str, tuple[str, ...]], ...] = ( + ("ix_accounts_username_id", "accounts", ("username", "id")), + ("ix_audit_logs_target_id", "audit_logs", ("target", "id")), +) + + +def _indexes(table: str) -> set[str]: + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table(table): + return set() + return { + str(index["name"]) + for index in inspector.get_indexes(table) + if index.get("name") + } + + +def upgrade() -> None: + for name, table, columns in INDEXES: + if name not in _indexes(table): + op.create_index(name, table, list(columns), unique=False) + + +def downgrade() -> None: + for name, table, _columns in reversed(INDEXES): + if name in _indexes(table): + op.drop_index(name, table_name=table) diff --git a/web/backend/models.py b/web/backend/models.py index d7afbd9..58489e7 100644 --- a/web/backend/models.py +++ b/web/backend/models.py @@ -259,6 +259,9 @@ class Account(Base): archived_at: Mapped[datetime | None] = mapped_column(UTCDateTime, nullable=True) __table_args__ = ( + # Account-name lookup is used by imports, task creation and support + # searches. Usernames are not unique in legacy data. + Index("ix_accounts_username_id", "username", "id"), # 巡检按「最久没查的优先」选行 Index("ix_accounts_ck_checked_at", "ck_checked_at"), # 重登按状态 + 退避时间选行 @@ -1649,6 +1652,8 @@ class AuditLog(Base): success: Mapped[bool | None] = mapped_column(Boolean, nullable=True, index=True) created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=_utcnow) + __table_args__ = (Index("ix_audit_logs_target_id", "target", "id"),) + class AccountEvent(Base): """账号事件流(append-only)。 diff --git a/web/backend/routers/douyu.py b/web/backend/routers/douyu.py index fd49336..fa27ba9 100644 --- a/web/backend/routers/douyu.py +++ b/web/backend/routers/douyu.py @@ -291,6 +291,26 @@ def _visible_tasks_query(db: Session, current: User): raise HTTPException(status_code=403, detail="无权查看斗鱼任务") +def _split_task_scope_query(query, handbook_scope: str | None) -> list: + """Build index-friendly branches for a workbench scope. + + Legacy rows map to the requested workbench by task type. A single + ``task_type IN (...) ORDER BY id DESC`` query makes MySQL scan the broad + handbook-scope index, so use one range per task type instead. + """ + if not handbook_scope: + return [query] + branches = [query.filter(DouyuTask.handbook_scope == handbook_scope)] + branches.extend( + query.filter( + DouyuTask.handbook_scope == "legacy", + DouyuTask.task_type == task_type, + ) + for task_type in DOUYU_HANDBOOK_TASK_TYPES[handbook_scope] + ) + return branches + + def _list_visible_tasks_for_support( db: Session, current: User, @@ -305,25 +325,13 @@ def _list_visible_tasks_for_support( (DouyuTask.account_assigned_to == current.id,), (DouyuTask.created_by == current.id,), ] - scope_filters: list[tuple[Any, ...]] - if handbook_scope: - scope_filters = [ - (DouyuTask.handbook_scope == handbook_scope,), - ( - DouyuTask.handbook_scope == "legacy", - DouyuTask.task_type.in_(DOUYU_HANDBOOK_TASK_TYPES[handbook_scope]), - ), - ] - else: - scope_filters = [()] - take = offset + limit task_ids: list[int] = [] for owner_filter in owner_filters: - for scope_filter in scope_filters: - query = db.query(DouyuTask.id).filter(*owner_filter, *scope_filter) - if batch_id: - query = query.filter(DouyuTask.batch_id == batch_id) + base_query = db.query(DouyuTask.id).filter(*owner_filter) + if batch_id: + base_query = base_query.filter(DouyuTask.batch_id == batch_id) + for query in _split_task_scope_query(base_query, handbook_scope): rows = query.order_by(DouyuTask.id.desc()).limit(take).all() task_ids.extend(int(row[0]) for row in rows) @@ -359,23 +367,12 @@ def _list_visible_tasks_for_admin( offset: int = 0, ) -> list[DouyuTask]: """管理员任务列表按作用域拆索引分支,避免 OR + JOIN 触发大排序。""" - if handbook_scope: - scope_filters: list[tuple[Any, ...]] = [ - (DouyuTask.handbook_scope == handbook_scope,), - ( - DouyuTask.handbook_scope == "legacy", - DouyuTask.task_type.in_(DOUYU_HANDBOOK_TASK_TYPES[handbook_scope]), - ), - ] - else: - scope_filters = [()] - take = offset + limit task_ids: list[int] = [] - for scope_filter in scope_filters: - query = db.query(DouyuTask.id).filter(*scope_filter) - if batch_id: - query = query.filter(DouyuTask.batch_id == batch_id) + base_query = db.query(DouyuTask.id) + if batch_id: + base_query = base_query.filter(DouyuTask.batch_id == batch_id) + for query in _split_task_scope_query(base_query, handbook_scope): rows = query.order_by(DouyuTask.id.desc()).limit(take).all() task_ids.extend(int(row[0]) for row in rows)