From 9497e1ace6ead2ed4855ff2292e6bc8be75d4c52 Mon Sep 17 00:00:00 2001 From: yml2213 Date: Sun, 30 Aug 2026 18:26:54 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=20MySQL=20=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E5=B9=B6=E6=8E=A5=E5=85=A5=20pytest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 4 ++ docker-compose.yml | 5 ++ pyproject.toml | 8 +++ uv.lock | 53 ++++++++++++++++++- .../20260830_0031_mysql_task_query_indexes.py | 52 ++++++++++++++++++ web/backend/routers/accounts.py | 17 +++--- web/backend/routers/cookies.py | 15 ++++-- web/backend/routers/dashboard.py | 17 +++--- web/backend/routers/douyu.py | 22 ++++++-- web/backend/routers/login.py | 9 +++- 10 files changed, 178 insertions(+), 24 deletions(-) create mode 100644 web/backend/migrations/versions/20260830_0031_mysql_task_query_indexes.py diff --git a/.env.example b/.env.example index 86d83dd..fddbd50 100644 --- a/.env.example +++ b/.env.example @@ -42,6 +42,10 @@ MYSQL_PASSWORD= MYSQL_ROOT_PASSWORD= MYSQL_BIND_HOST=127.0.0.1 MYSQL_HOST_PORT=3307 +# MySQL 性能观测；同机部署资源紧张时可先使用 256M。 +MYSQL_INNODB_BUFFER_POOL_SIZE=512M +MYSQL_SLOW_QUERY_LOG=1 +MYSQL_LONG_QUERY_TIME=1 # 本地调试后端连接 MySQL（./dev.sh 使用；默认连接上面的宿主机端口） DEV_DB_HOST=127.0.0.1 diff --git a/docker-compose.yml b/docker-compose.yml index eb8fb10..9876808 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,6 +93,11 @@ services: command: - --character-set-server=utf8mb4 - --collation-server=utf8mb4_0900_ai_ci + # 业务表已超过 300MiB；可通过 .env 调整，默认给 MySQL 512MiB 缓冲池。 + - --innodb-buffer-pool-size=${MYSQL_INNODB_BUFFER_POOL_SIZE:-512M} + # 默认记录 1 秒以上查询，便于验证查询优化；稳定后可设 MYSQL_SLOW_QUERY_LOG=0。 + - --slow-query-log=${MYSQL_SLOW_QUERY_LOG:-1} + - --long-query-time=${MYSQL_LONG_QUERY_TIME:-1} healthcheck: test: ["CMD-SHELL", "mysqladmin ping -h localhost -uroot -p$$MYSQL_ROOT_PASSWORD --silent"] interval: 10s diff --git a/pyproject.toml b/pyproject.toml index aa156cc..064fd4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,3 +34,11 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["core", "utils", "web"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[dependency-groups] +dev = [ + "pytest>=9.1.1", +] diff --git a/uv.lock b/uv.lock index b8ac4f3..84fa56b 100644 --- a/uv.lock +++ b/uv.lock @@ -226,7 +226,7 @@ wheels = [ [[package]] name = "douyu-login-py" -version = "0.2.0" +version = "0.2.1" source = { editable = "." } dependencies = [ { name = "alembic" }, @@ -252,6 +252,11 @@ dependencies = [ { name = "websockets" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + [package.metadata] requires-dist = [ { name = "alembic", specifier = ">=1.18.4" }, @@ -277,6 +282,9 @@ requires-dist = [ { name = "websockets", specifier = ">=16.0" }, ] +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=9.1.1" }] + [[package]] name = "ecdsa" version = "0.19.2" @@ -362,6 +370,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "loguru" version = "0.7.3" @@ -489,6 +506,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/e1/542a474affab20fd4a0f1836cb234e8493519da6b76899e30bcc5d990b8b/pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414", size = 2463612, upload-time = "2026-04-01T14:43:39.421Z" }, ] +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "protobuf" version = "7.35.1" @@ -595,6 +621,15 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/ba/8e/aedef81641c8dca6fd0fb7294de5bed9c45f3397d67fddf755c1042c2642/PyExecJS-1.5.1.tar.gz", hash = "sha256:34cc1d070976918183ff7bdc0ad71f8157a891c92708c00c5fbbff7a769f505c", size = 13344, upload-time = "2018-01-18T04:33:55.126Z" } +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + [[package]] name = "pymysql" version = "1.2.0" @@ -613,6 +648,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" }, ] +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.2" diff --git a/web/backend/migrations/versions/20260830_0031_mysql_task_query_indexes.py b/web/backend/migrations/versions/20260830_0031_mysql_task_query_indexes.py new file mode 100644 index 0000000..0a8fe86 --- /dev/null +++ b/web/backend/migrations/versions/20260830_0031_mysql_task_query_indexes.py @@ -0,0 +1,52 @@ +"""Add composite indexes for task polling, ordering, and retention queries. + +Revision ID: 20260830_0031 +Revises: 20260829_0030 +Create Date: 2026-08-30 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "20260830_0031" +down_revision: Union[str, None] = "20260829_0030" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +INDEXES = ( + # /api/douyu/tasks filters by scope and returns the newest rows first. + ("ix_douyu_tasks_handbook_scope_id", "douyu_tasks", ["handbook_scope", "id"]), + # The legacy compatibility branch also filters by task_type. + ( + "ix_douyu_tasks_handbook_scope_task_type_id", + "douyu_tasks", + ["handbook_scope", "task_type", "id"], + ), + # Supports terminal-task retention and finished-time ordered Cookie views. + ("ix_douyu_tasks_status_finished_at_id", "douyu_tasks", ["status", "finished_at", "id"]), + ("ix_login_tasks_status_finished_at_id", "login_tasks", ["status", "finished_at", "id"]), +) + + +def _indexes(bind, table_name: str) -> set[str]: + if not sa.inspect(bind).has_table(table_name): + return set() + return {index["name"] for index in sa.inspect(bind).get_indexes(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + for name, table_name, columns in INDEXES: + if name not in _indexes(bind, table_name): + op.create_index(name, table_name, columns) + + +def downgrade() -> None: + bind = op.get_bind() + for name, table_name, _ in reversed(INDEXES): + if name in _indexes(bind, table_name): + op.drop_index(name, table_name=table_name) diff --git a/web/backend/routers/accounts.py b/web/backend/routers/accounts.py index d6a44a6..02d0ccd 100644 --- a/web/backend/routers/accounts.py +++ b/web/backend/routers/accounts.py @@ -126,7 +126,7 @@ def list_accounts( total = None if page is not None: - total = query.order_by(None).count() + total = query.order_by(None).with_entities(func.count(Account.id)).scalar() or 0 query = query.order_by(Account.id) if page is not None: query = query.offset((page - 1) * page_size).limit(page_size) @@ -173,13 +173,18 @@ def accounts_summary( else: raise HTTPException(status_code=403, detail="无权查看账号") - total = query.count() - assigned_count = query.filter(Account.assigned_to.isnot(None)).count() + total = query.with_entities(func.count(Account.id)).scalar() or 0 + assigned_count = ( + query.filter(Account.assigned_to.isnot(None)) + .with_entities(func.count(Account.id)) + .scalar() + or 0 + ) tag_count = ( query.filter(Account.tag != "", Account.tag.isnot(None)) - .with_entities(Account.tag) - .distinct() - .count() + .with_entities(func.count(func.distinct(Account.tag))) + .scalar() + or 0 ) return { "total": total, diff --git a/web/backend/routers/cookies.py b/web/backend/routers/cookies.py index 019fbc1..97a79e3 100644 --- a/web/backend/routers/cookies.py +++ b/web/backend/routers/cookies.py @@ -5,7 +5,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import StreamingResponse -from sqlalchemy import case, or_ +from sqlalchemy import case, func, or_ from sqlalchemy.orm import Session, defer, joinedload import io import csv @@ -145,7 +145,7 @@ def list_cookies( total = None if page is not None: - total = query.order_by(None).count() + total = query.order_by(None).with_entities(func.count(LoginTask.id)).scalar() or 0 query = _order_cookie_tasks(query, selected_names) if page is not None: query = query.offset((page - 1) * page_size).limit(page_size) @@ -209,8 +209,13 @@ def cookies_summary( query = _visible_cookie_tasks_query(db, current) if user_has_permission(current, "login:view_all"): query = query.join(Account, LoginTask.account_id == Account.id) - total = query.count() - assigned_count = query.filter(Account.assigned_to.isnot(None)).count() + total = query.with_entities(func.count(LoginTask.id)).scalar() or 0 + assigned_count = ( + query.filter(Account.assigned_to.isnot(None)) + .with_entities(func.count(LoginTask.id)) + .scalar() + or 0 + ) return { "total": total, "assigned_count": assigned_count, @@ -242,7 +247,7 @@ def list_cookie_operations( Account.tag.ilike(f"%{search_text}%"), )) - total = query.order_by(None).count() + total = query.order_by(None).with_entities(func.count(LoginTask.id)).scalar() or 0 tasks = ( query.order_by(LoginTask.finished_at.desc(), LoginTask.id.desc()) .offset((page - 1) * page_size) diff --git a/web/backend/routers/dashboard.py b/web/backend/routers/dashboard.py index dd92681..0c49a97 100644 --- a/web/backend/routers/dashboard.py +++ b/web/backend/routers/dashboard.py @@ -55,9 +55,14 @@ def dashboard_summary( """返回首页所需的全部轻量统计。""" douyu_accounts = 0 if user_has_permission(current, "account:view_all"): - douyu_accounts = db.query(Account).count() + douyu_accounts = db.query(func.count(Account.id)).scalar() or 0 elif user_has_permission(current, "account:view_assigned"): - douyu_accounts = db.query(Account).filter(Account.assigned_to == current.id).count() + douyu_accounts = ( + db.query(func.count(Account.id)) + .filter(Account.assigned_to == current.id) + .scalar() + or 0 + ) login_tasks = _empty_task_summary() if any(user_has_permission(current, permission) for permission in ( @@ -79,7 +84,7 @@ def dashboard_summary( ) if not user_has_permission(current, "login:view_all"): cookie_query = cookie_query.filter(Account.assigned_to == current.id) - cookies = cookie_query.count() + cookies = cookie_query.with_entities(func.count(LoginTask.id)).scalar() or 0 huya_accounts = 0 can_view_huya_accounts = any(user_has_permission(current, permission) for permission in ( @@ -91,7 +96,7 @@ def dashboard_summary( huya_account_query = db.query(HuyaAccount) if not _can_view_huya_all(current): huya_account_query = huya_account_query.filter(HuyaAccount.assigned_to == current.id) - huya_accounts = huya_account_query.count() + huya_accounts = huya_account_query.with_entities(func.count(HuyaAccount.id)).scalar() or 0 huya_tasks = _empty_task_summary() huya_goods = 0 @@ -101,8 +106,8 @@ def dashboard_summary( if not _can_view_huya_all(current): huya_task_query = huya_task_query.filter(HuyaAccount.assigned_to == current.id) huya_tasks = _task_summary(huya_task_query, HuyaTask) - huya_goods = db.query(HuyaGoodsSnapshot).count() - huya_recharge_goods = db.query(HuyaRechargeGoodsSnapshot).count() + huya_goods = db.query(func.count(HuyaGoodsSnapshot.id)).scalar() or 0 + huya_recharge_goods = db.query(func.count(HuyaRechargeGoodsSnapshot.id)).scalar() or 0 return { "douyu": { diff --git a/web/backend/routers/douyu.py b/web/backend/routers/douyu.py index 8cd7e9c..699f8d1 100644 --- a/web/backend/routers/douyu.py +++ b/web/backend/routers/douyu.py @@ -8,8 +8,8 @@ from datetime import datetime, timezone from fastapi import APIRouter, Depends, HTTPException, Query, Request, WebSocket, WebSocketDisconnect from loguru import logger -from sqlalchemy import or_ -from sqlalchemy.orm import Session, joinedload +from sqlalchemy import func, or_ +from sqlalchemy.orm import Session, joinedload, load_only from ..database import SessionLocal, get_db from ..deps import authenticate_websocket, get_current_user, require_permission @@ -152,7 +152,15 @@ def _visible_task_accounts_query(db: Session, current: User): def _visible_tasks_query(db: Session, current: User): """返回当前用户可查看的斗鱼任务查询。""" - query = db.query(DouyuTask).options(joinedload(DouyuTask.account)) + # 任务列表只展示账号识别信息，避免 joinedload 把密码、邮箱等加密大字段带出。 + account_loader = joinedload(DouyuTask.account).load_only( + Account.id, + Account.username, + Account.uid, + Account.nickname, + Account.assigned_to, + ) + query = db.query(DouyuTask).options(account_loader) if _can_view_all(current): return query if user_has_permission(current, "account:view_assigned"): @@ -589,7 +597,13 @@ def list_tasks( )) total = None if page is not None: - total = query.enable_eagerloads(False).order_by(None).count() + total = ( + query.enable_eagerloads(False) + .order_by(None) + .with_entities(func.count(DouyuTask.id)) + .scalar() + or 0 + ) query = query.order_by(DouyuTask.id.desc()) if page is not None: query = query.offset((page - 1) * page_size).limit(page_size) diff --git a/web/backend/routers/login.py b/web/backend/routers/login.py index c22d879..9ea855c 100644 --- a/web/backend/routers/login.py +++ b/web/backend/routers/login.py @@ -5,7 +5,7 @@ import threading from datetime import datetime from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect from sqlalchemy import func -from sqlalchemy.orm import Session, defer +from sqlalchemy.orm import Session, defer, load_only from ..database import get_db, SessionLocal from ..models import User, Account, LoginTask, ProxyConfig as ProxyConfigModel @@ -120,7 +120,12 @@ def list_tasks( account_ids = [t.account_id for t in rows] accounts_map = {} if account_ids: - accs = db.query(Account).filter(Account.id.in_(account_ids)).all() + accs = ( + db.query(Account) + .options(load_only(Account.id, Account.username)) + .filter(Account.id.in_(account_ids)) + .all() + ) accounts_map = {a.id: a.username for a in accs} result = []