perf: 优化 MySQL 查询并接入 pytest

2026-08-30 18:26:54 +08:00
parent 1413b4f030
commit 9497e1ace6
10 changed files with 178 additions and 24 deletions
@@ -42,6 +42,10 @@ MYSQL_PASSWORD=
 MYSQL_ROOT_PASSWORD=
 MYSQL_BIND_HOST=127.0.0.1
 MYSQL_HOST_PORT=3307
+# MySQL 性能观测；同机部署资源紧张时可先使用 256M。
+MYSQL_INNODB_BUFFER_POOL_SIZE=512M
+MYSQL_SLOW_QUERY_LOG=1
+MYSQL_LONG_QUERY_TIME=1

 # 本地调试后端连接 MySQL（./dev.sh 使用；默认连接上面的宿主机端口）
 DEV_DB_HOST=127.0.0.1
@@ -93,6 +93,11 @@ services:
    command:
      - --character-set-server=utf8mb4
      - --collation-server=utf8mb4_0900_ai_ci
+      # 业务表已超过 300MiB；可通过 .env 调整，默认给 MySQL 512MiB 缓冲池。
+      - --innodb-buffer-pool-size=${MYSQL_INNODB_BUFFER_POOL_SIZE:-512M}
+      # 默认记录 1 秒以上查询，便于验证查询优化；稳定后可设 MYSQL_SLOW_QUERY_LOG=0。
+      - --slow-query-log=${MYSQL_SLOW_QUERY_LOG:-1}
+      - --long-query-time=${MYSQL_LONG_QUERY_TIME:-1}
    healthcheck:
      test: ["CMD-SHELL", "mysqladmin ping -h localhost -uroot -p$$MYSQL_ROOT_PASSWORD --silent"]
      interval: 10s
@@ -34,3 +34,11 @@ build-backend = "hatchling.build"

 [tool.hatch.build.targets.wheel]
 packages = ["core", "utils", "web"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[dependency-groups]
+dev = [
+    "pytest>=9.1.1",
+]
@@ -226,7 +226,7 @@ wheels = [

 [[package]]
 name = "douyu-login-py"
-version = "0.2.0"
+version = "0.2.1"
 source = { editable = "." }
 dependencies = [
    { name = "alembic" },
@@ -252,6 +252,11 @@ dependencies = [
    { name = "websockets" },
 ]

+[package.dev-dependencies]
+dev = [
+    { name = "pytest" },
+]
+
 [package.metadata]
 requires-dist = [
    { name = "alembic", specifier = ">=1.18.4" },
@@ -277,6 +282,9 @@ requires-dist = [
    { name = "websockets", specifier = ">=16.0" },
 ]

+[package.metadata.requires-dev]
+dev = [{ name = "pytest", specifier = ">=9.1.1" }]
+
 [[package]]
 name = "ecdsa"
 version = "0.19.2"
@@ -362,6 +370,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" },
 ]

+[[package]]
+name = "iniconfig"
+version = "2.3.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
+]
+
 [[package]]
 name = "loguru"
 version = "0.7.3"
@@ -489,6 +506,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/10/e1/542a474affab20fd4a0f1836cb234e8493519da6b76899e30bcc5d990b8b/pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414", size = 2463612, upload-time = "2026-04-01T14:43:39.421Z" },
 ]

+[[package]]
+name = "pluggy"
+version = "1.6.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
+]
+
 [[package]]
 name = "protobuf"
 version = "7.35.1"
@@ -595,6 +621,15 @@ dependencies = [
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/ba/8e/aedef81641c8dca6fd0fb7294de5bed9c45f3397d67fddf755c1042c2642/PyExecJS-1.5.1.tar.gz", hash = "sha256:34cc1d070976918183ff7bdc0ad71f8157a891c92708c00c5fbbff7a769f505c", size = 13344, upload-time = "2018-01-18T04:33:55.126Z" }

+[[package]]
+name = "pygments"
+version = "2.21.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" },
+]
+
 [[package]]
 name = "pymysql"
 version = "1.2.0"
@@ -613,6 +648,22 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" },
 ]

+[[package]]
+name = "pytest"
+version = "9.1.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+    { name = "iniconfig" },
+    { name = "packaging" },
+    { name = "pluggy" },
+    { name = "pygments" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
+]
+
 [[package]]
 name = "python-dotenv"
 version = "1.2.2"
@@ -0,0 +1,52 @@
+"""Add composite indexes for task polling, ordering, and retention queries.
+
+Revision ID: 20260830_0031
+Revises: 20260829_0030
+Create Date: 2026-08-30
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision: str = "20260830_0031"
+down_revision: Union[str, None] = "20260829_0030"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+INDEXES = (
+    # /api/douyu/tasks filters by scope and returns the newest rows first.
+    ("ix_douyu_tasks_handbook_scope_id", "douyu_tasks", ["handbook_scope", "id"]),
+    # The legacy compatibility branch also filters by task_type.
+    (
+        "ix_douyu_tasks_handbook_scope_task_type_id",
+        "douyu_tasks",
+        ["handbook_scope", "task_type", "id"],
+    ),
+    # Supports terminal-task retention and finished-time ordered Cookie views.
+    ("ix_douyu_tasks_status_finished_at_id", "douyu_tasks", ["status", "finished_at", "id"]),
+    ("ix_login_tasks_status_finished_at_id", "login_tasks", ["status", "finished_at", "id"]),
+)
+
+
+def _indexes(bind, table_name: str) -> set[str]:
+    if not sa.inspect(bind).has_table(table_name):
+        return set()
+    return {index["name"] for index in sa.inspect(bind).get_indexes(table_name)}
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    for name, table_name, columns in INDEXES:
+        if name not in _indexes(bind, table_name):
+            op.create_index(name, table_name, columns)
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    for name, table_name, _ in reversed(INDEXES):
+        if name in _indexes(bind, table_name):
+            op.drop_index(name, table_name=table_name)
@@ -126,7 +126,7 @@ def list_accounts(

    total = None
    if page is not None:
-        total = query.order_by(None).count()
+        total = query.order_by(None).with_entities(func.count(Account.id)).scalar() or 0
    query = query.order_by(Account.id)
    if page is not None:
        query = query.offset((page - 1) * page_size).limit(page_size)
@@ -173,13 +173,18 @@ def accounts_summary(
        else:
            raise HTTPException(status_code=403, detail="无权查看账号")

-    total = query.count()
-    assigned_count = query.filter(Account.assigned_to.isnot(None)).count()
+    total = query.with_entities(func.count(Account.id)).scalar() or 0
+    assigned_count = (
+        query.filter(Account.assigned_to.isnot(None))
+        .with_entities(func.count(Account.id))
+        .scalar()
+        or 0
+    )
    tag_count = (
        query.filter(Account.tag != "", Account.tag.isnot(None))
-        .with_entities(Account.tag)
-        .distinct()
-        .count()
+        .with_entities(func.count(func.distinct(Account.tag)))
+        .scalar()
+        or 0
    )
    return {
        "total": total,
@@ -5,7 +5,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timezone
 from fastapi import APIRouter, Depends, HTTPException, Query
 from fastapi.responses import StreamingResponse
-from sqlalchemy import case, or_
+from sqlalchemy import case, func, or_
 from sqlalchemy.orm import Session, defer, joinedload
 import io
 import csv
@@ -145,7 +145,7 @@ def list_cookies(

    total = None
    if page is not None:
-        total = query.order_by(None).count()
+        total = query.order_by(None).with_entities(func.count(LoginTask.id)).scalar() or 0
    query = _order_cookie_tasks(query, selected_names)
    if page is not None:
        query = query.offset((page - 1) * page_size).limit(page_size)
@@ -209,8 +209,13 @@ def cookies_summary(
    query = _visible_cookie_tasks_query(db, current)
    if user_has_permission(current, "login:view_all"):
        query = query.join(Account, LoginTask.account_id == Account.id)
-    total = query.count()
-    assigned_count = query.filter(Account.assigned_to.isnot(None)).count()
+    total = query.with_entities(func.count(LoginTask.id)).scalar() or 0
+    assigned_count = (
+        query.filter(Account.assigned_to.isnot(None))
+        .with_entities(func.count(LoginTask.id))
+        .scalar()
+        or 0
+    )
    return {
        "total": total,
        "assigned_count": assigned_count,
@@ -242,7 +247,7 @@ def list_cookie_operations(
            Account.tag.ilike(f"%{search_text}%"),
        ))

-    total = query.order_by(None).count()
+    total = query.order_by(None).with_entities(func.count(LoginTask.id)).scalar() or 0
    tasks = (
        query.order_by(LoginTask.finished_at.desc(), LoginTask.id.desc())
        .offset((page - 1) * page_size)
@@ -55,9 +55,14 @@ def dashboard_summary(
    """返回首页所需的全部轻量统计。"""
    douyu_accounts = 0
    if user_has_permission(current, "account:view_all"):
-        douyu_accounts = db.query(Account).count()
+        douyu_accounts = db.query(func.count(Account.id)).scalar() or 0
    elif user_has_permission(current, "account:view_assigned"):
-        douyu_accounts = db.query(Account).filter(Account.assigned_to == current.id).count()
+        douyu_accounts = (
+            db.query(func.count(Account.id))
+            .filter(Account.assigned_to == current.id)
+            .scalar()
+            or 0
+        )

    login_tasks = _empty_task_summary()
    if any(user_has_permission(current, permission) for permission in (
@@ -79,7 +84,7 @@ def dashboard_summary(
        )
        if not user_has_permission(current, "login:view_all"):
            cookie_query = cookie_query.filter(Account.assigned_to == current.id)
-        cookies = cookie_query.count()
+        cookies = cookie_query.with_entities(func.count(LoginTask.id)).scalar() or 0

    huya_accounts = 0
    can_view_huya_accounts = any(user_has_permission(current, permission) for permission in (
@@ -91,7 +96,7 @@ def dashboard_summary(
        huya_account_query = db.query(HuyaAccount)
        if not _can_view_huya_all(current):
            huya_account_query = huya_account_query.filter(HuyaAccount.assigned_to == current.id)
-        huya_accounts = huya_account_query.count()
+        huya_accounts = huya_account_query.with_entities(func.count(HuyaAccount.id)).scalar() or 0

    huya_tasks = _empty_task_summary()
    huya_goods = 0
@@ -101,8 +106,8 @@ def dashboard_summary(
        if not _can_view_huya_all(current):
            huya_task_query = huya_task_query.filter(HuyaAccount.assigned_to == current.id)
        huya_tasks = _task_summary(huya_task_query, HuyaTask)
-        huya_goods = db.query(HuyaGoodsSnapshot).count()
-        huya_recharge_goods = db.query(HuyaRechargeGoodsSnapshot).count()
+        huya_goods = db.query(func.count(HuyaGoodsSnapshot.id)).scalar() or 0
+        huya_recharge_goods = db.query(func.count(HuyaRechargeGoodsSnapshot.id)).scalar() or 0

    return {
        "douyu": {
@@ -8,8 +8,8 @@ from datetime import datetime, timezone

 from fastapi import APIRouter, Depends, HTTPException, Query, Request, WebSocket, WebSocketDisconnect
 from loguru import logger
-from sqlalchemy import or_
-from sqlalchemy.orm import Session, joinedload
+from sqlalchemy import func, or_
+from sqlalchemy.orm import Session, joinedload, load_only

 from ..database import SessionLocal, get_db
 from ..deps import authenticate_websocket, get_current_user, require_permission
@@ -152,7 +152,15 @@ def _visible_task_accounts_query(db: Session, current: User):

 def _visible_tasks_query(db: Session, current: User):
    """返回当前用户可查看的斗鱼任务查询。"""
-    query = db.query(DouyuTask).options(joinedload(DouyuTask.account))
+    # 任务列表只展示账号识别信息，避免 joinedload 把密码、邮箱等加密大字段带出。
+    account_loader = joinedload(DouyuTask.account).load_only(
+        Account.id,
+        Account.username,
+        Account.uid,
+        Account.nickname,
+        Account.assigned_to,
+    )
+    query = db.query(DouyuTask).options(account_loader)
    if _can_view_all(current):
        return query
    if user_has_permission(current, "account:view_assigned"):
@@ -589,7 +597,13 @@ def list_tasks(
        ))
    total = None
    if page is not None:
-        total = query.enable_eagerloads(False).order_by(None).count()
+        total = (
+            query.enable_eagerloads(False)
+            .order_by(None)
+            .with_entities(func.count(DouyuTask.id))
+            .scalar()
+            or 0
+        )
    query = query.order_by(DouyuTask.id.desc())
    if page is not None:
        query = query.offset((page - 1) * page_size).limit(page_size)
@@ -5,7 +5,7 @@ import threading
 from datetime import datetime
 from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect
 from sqlalchemy import func
-from sqlalchemy.orm import Session, defer
+from sqlalchemy.orm import Session, defer, load_only

 from ..database import get_db, SessionLocal
 from ..models import User, Account, LoginTask, ProxyConfig as ProxyConfigModel
@@ -120,7 +120,12 @@ def list_tasks(
    account_ids = [t.account_id for t in rows]
    accounts_map = {}
    if account_ids:
-        accs = db.query(Account).filter(Account.id.in_(account_ids)).all()
+        accs = (
+            db.query(Account)
+            .options(load_only(Account.id, Account.username))
+            .filter(Account.id.in_(account_ids))
+            .all()
+        )
        accounts_map = {a.id: a.username for a in accs}

    result = []