From 32f9eadfdb420f33706e21e22c574680c220a647 Mon Sep 17 00:00:00 2001 From: yml2213 Date: Sat, 25 Jul 2026 10:23:19 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81=20MySQL=20=E8=BF=81=E7=A7=BB?= =?UTF-8?q?=E4=B8=8E=E6=80=A7=E8=83=BD=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 18 ++ Dockerfile | 45 ++-- deploy.sh | 54 ++++- docker-compose.yml | 49 ++++- pyproject.toml | 1 + scripts/migrate_sqlite_to_mysql.py | 192 ++++++++++++++++++ uv.lock | 11 + web/backend/database.py | 56 ++++- ...60725_0009_database_performance_indexes.py | 67 ++++++ 9 files changed, 460 insertions(+), 33 deletions(-) create mode 100644 scripts/migrate_sqlite_to_mysql.py create mode 100644 web/backend/migrations/versions/20260725_0009_database_performance_indexes.py diff --git a/.env.example b/.env.example index eb4865f..721dd55 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,24 @@ APP_ENCRYPTION_KEY= ADMIN_USERNAME=admin ADMIN_PASSWORD=admin123 +# Docker 对外端口（容器内仍监听 8800；服务器当前使用 8000） +APP_PORT=8000 + +# Docker 构建镜像源（服务器保持 true；本地测试可改为 false） +USE_CHINA_MIRRORS=true + +# MySQL 配置（生产环境必须设置密码；MySQL 不映射宿主端口） +MYSQL_DATABASE=douyu_login +MYSQL_USER=douyu_login +MYSQL_PASSWORD= +MYSQL_ROOT_PASSWORD= + +# 数据库连接池配置（仅 MySQL 生效） +DB_POOL_SIZE=20 +DB_MAX_OVERFLOW=20 +DB_POOL_TIMEOUT=30 +DB_POOL_RECYCLE=1800 + # Cookie 安全标志（生产环境 HTTPS 部署时设为 true） COOKIE_SECURE=false diff --git a/Dockerfile b/Dockerfile index a6c956d..eb36a8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,38 +13,53 @@ RUN npm run build # ===== 阶段2: Python 运行时 ===== FROM python:3.12-slim AS runtime -# 优先阿里云内网镜像，其次公网阿里云，最后官方源。 -RUN cp /etc/apt/sources.list.d/debian.sources /tmp/debian.sources && \ - sed -i 's/deb.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list.d/debian.sources && \ - (apt-get update -o APT::Update::Error-Mode=any || \ - (sed -i 's/mirrors.cloud.aliyuncs.com/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources && \ - apt-get update -o APT::Update::Error-Mode=any) || \ - (cp /tmp/debian.sources /etc/apt/sources.list.d/debian.sources && \ - apt-get update -o APT::Update::Error-Mode=any)) && \ +ARG USE_CHINA_MIRRORS=true + +# 服务器默认使用阿里云镜像；本地构建可设 USE_CHINA_MIRRORS=false 直连官方源。 +RUN if [ "$USE_CHINA_MIRRORS" = "true" ]; then \ + cp /etc/apt/sources.list.d/debian.sources /tmp/debian.sources && \ + sed -i 's/deb.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list.d/debian.sources && \ + (apt-get update -o APT::Update::Error-Mode=any || \ + (sed -i 's/mirrors.cloud.aliyuncs.com/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources && \ + apt-get update -o APT::Update::Error-Mode=any) || \ + (cp /tmp/debian.sources /etc/apt/sources.list.d/debian.sources && \ + apt-get update -o APT::Update::Error-Mode=any)); \ + else \ + apt-get update -o APT::Update::Error-Mode=any; \ + fi && \ apt-get install -y --no-install-recommends \ libgl1 \ libglib2.0-0 \ nodejs \ && rm -rf /var/lib/apt/lists/* -# 安装 uv：优先阿里云内网 PyPI → 公网阿里云 → 官方。 -RUN pip install -i https://mirrors.cloud.aliyuncs.com/pypi/simple/ uv || \ - pip install -i https://mirrors.aliyun.com/pypi/simple/ uv || \ - pip install uv +# 安装 uv：服务器走阿里云镜像，本地直连官方 PyPI。 +RUN if [ "$USE_CHINA_MIRRORS" = "true" ]; then \ + pip install -i https://mirrors.cloud.aliyuncs.com/pypi/simple/ uv || \ + pip install -i https://mirrors.aliyun.com/pypi/simple/ uv || \ + pip install uv; \ + else \ + pip install uv; \ + fi WORKDIR /app # 先复制依赖文件，利用 Docker 缓存层 COPY pyproject.toml uv.lock* README.md alembic.ini ./ RUN uv venv /app/.venv && \ - (UV_INDEX_URL=https://mirrors.cloud.aliyuncs.com/pypi/simple/ uv pip install -e . || \ - UV_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/ uv pip install -e . || \ - uv pip install -e .) + if [ "$USE_CHINA_MIRRORS" = "true" ]; then \ + (UV_INDEX_URL=https://mirrors.cloud.aliyuncs.com/pypi/simple/ uv pip install -e . || \ + UV_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/ uv pip install -e . || \ + uv pip install -e .); \ + else \ + uv pip install -e .; \ + fi # 复制项目代码 COPY core/ ./core/ COPY utils/ ./utils/ COPY web/backend/ ./web/backend/ +COPY scripts/ ./scripts/ # 复制前端构建产物 COPY --from=frontend-builder /build/dist ./web/frontend/dist diff --git a/deploy.sh b/deploy.sh index de0b884..12d816e 100755 --- a/deploy.sh +++ b/deploy.sh @@ -5,6 +5,7 @@ set -e ROOT_DIR="$(cd "$(dirname "$0")" && pwd)" cd "$ROOT_DIR" +APP_PORT="${APP_PORT:-8000}" # ── 工具函数 ────────────────────────────────────────────── @@ -44,9 +45,6 @@ cmd_deploy() { # 创建数据和日志目录 mkdir -p data logs - # 先停掉旧容器（避免名称冲突） - $COMPOSE down 2>/dev/null || true - # 构建并启动 echo "正在构建镜像（首次构建约3-5分钟）..." $COMPOSE build @@ -60,12 +58,12 @@ cmd_deploy() { # 健康检查 for i in $(seq 1 15); do - if curl -s http://localhost:8800/api/health | grep -q "ok" 2>/dev/null; then + if curl -s "http://localhost:${APP_PORT}/api/health" | grep -q "ok" 2>/dev/null; then echo "" echo "==============================" echo " ✅ 部署成功！" echo "==============================" - echo " 访问地址: http://localhost:8800" + echo " 访问地址: http://localhost:${APP_PORT}" echo " 默认账号见环境变量 ADMIN_USERNAME/ADMIN_PASSWORD（未设置则为 admin/admin123）" echo "" echo " 查看日志: $COMPOSE logs -f" @@ -92,7 +90,7 @@ cmd_reset() { echo "==============================" echo "" echo " 将删除以下内容：" - echo " • 数据库文件 (data/web.db)" + echo " • MySQL 数据卷" echo " • 历史Cookie (data/cookies/)" echo " • 应用日志 (logs/)" echo " • Docker容器和镜像" @@ -110,11 +108,11 @@ cmd_reset() { # 停止并删除容器 if [ -n "$COMPOSE" ]; then echo " 停止并删除容器..." - $COMPOSE down --rmi local 2>/dev/null || $COMPOSE down 2>/dev/null || true + $COMPOSE down --volumes --rmi local 2>/dev/null || $COMPOSE down --volumes 2>/dev/null || true fi # 删除数据文件 - echo " 删除数据库..." + echo " 删除旧 SQLite 数据..." rm -rf data/web.db data/web.db-shm data/web.db-wal echo " 删除Cookie..." @@ -143,6 +141,42 @@ cmd_logs() { $COMPOSE logs -f } +cmd_migrate_mysql() { + COMPOSE=$(detect_compose) + if [ -z "$COMPOSE" ]; then + echo "❌ 未检测到 docker compose" + exit 1 + fi + if [ ! -f data/web.db ]; then + echo "❌ 未找到 data/web.db，无法迁移 SQLite 数据" + exit 1 + fi + + echo "正在启动 MySQL..." + $COMPOSE up -d mysql + + echo "等待 MySQL 就绪..." + for i in $(seq 1 30); do + if $COMPOSE exec -T mysql sh -c 'mysqladmin ping -h localhost -uroot -p"$MYSQL_ROOT_PASSWORD" --silent' >/dev/null 2>&1; then + echo "MySQL 已就绪" + break + fi + if [ "$i" = "30" ]; then + echo "❌ MySQL 启动超时，查看日志：" + $COMPOSE logs --tail 50 mysql + exit 1 + fi + sleep 2 + done + + echo "正在构建迁移运行镜像..." + $COMPOSE build douyu-login + + echo "开始迁移 SQLite 到 MySQL..." + $COMPOSE run --rm --no-deps douyu-login \ + python scripts/migrate_sqlite_to_mysql.py --source /app/data/web.db +} + cmd_stop() { COMPOSE=$(detect_compose) if [ -z "$COMPOSE" ]; then @@ -180,13 +214,14 @@ cmd_help() { echo " 命令:" echo " (无参数) 构建并部署" echo " reset 删除所有数据并重置（需确认）" + echo " migrate-mysql 先启动 MySQL 并迁移 data/web.db" echo " logs 查看实时日志" echo " stop 停止服务" echo " restart 重启服务" echo " dev 本地调试启动（后端 reload + 前端热更新）" echo " help 显示帮助" echo "" - echo " 访问地址: http://localhost:8800" + echo " 访问地址: http://localhost:${APP_PORT}" echo " 调试地址: http://localhost:5174" echo " 默认账号见环境变量 ADMIN_USERNAME/ADMIN_PASSWORD" echo "" @@ -196,6 +231,7 @@ cmd_help() { case "${1:-}" in reset) cmd_reset ;; + migrate-mysql) cmd_migrate_mysql ;; logs) cmd_logs ;; stop) cmd_stop ;; restart) cmd_restart ;; diff --git a/docker-compose.yml b/docker-compose.yml index 3b8476f..cb67517 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,31 @@ services: douyu-login: - build: . + build: + context: . + args: + # 服务器默认走阿里云镜像；本地测试设为 false。 + USE_CHINA_MIRRORS: ${USE_CHINA_MIRRORS:-true} container_name: douyu-login restart: unless-stopped ports: - - "8800:8800" + # 保持现有服务端口；可通过 APP_PORT 覆盖。 + - "${APP_PORT:-8000}:8800" volumes: - # 持久化数据库和日志 + # 持久化运行时文件和日志，数据库由 mysql-data 卷管理。 - ./data:/app/data - ./logs:/app/logs environment: - TZ=Asia/Shanghai + # MySQL 仅在 Compose 内网开放，不映射宿主机端口。 + - DB_HOST=mysql + - DB_PORT=3306 + - DB_NAME=${MYSQL_DATABASE:-douyu_login} + - DB_USER=${MYSQL_USER:-douyu_login} + - DB_PASSWORD=${MYSQL_PASSWORD:?请在 .env 中设置 MYSQL_PASSWORD} + - DB_POOL_SIZE=${DB_POOL_SIZE:-20} + - DB_MAX_OVERFLOW=${DB_MAX_OVERFLOW:-20} + - DB_POOL_TIMEOUT=${DB_POOL_TIMEOUT:-30} + - DB_POOL_RECYCLE=${DB_POOL_RECYCLE:-1800} # JWT 密钥（生产环境务必修改，可用 python -c "import secrets; print(secrets.token_urlsafe(32))" 生成） - JWT_SECRET_KEY=${JWT_SECRET_KEY:-} # 敏感字段落盘加密密钥（生产环境务必长期保持不变） @@ -26,6 +41,9 @@ services: - MAIL_ROUNDCUBE_URL=${MAIL_ROUNDCUBE_URL:-} # Uvicorn reload（生产环境保持 false） - UVICORN_RELOAD=${UVICORN_RELOAD:-false} + depends_on: + mysql: + condition: service_healthy # 增加文件描述符限制，避免 "Too many open files" 错误 ulimits: nofile: @@ -36,3 +54,28 @@ services: interval: 30s timeout: 5s retries: 3 + + mysql: + image: docker.m.daocloud.io/library/mysql:8.4 + container_name: douyu-login-db + restart: unless-stopped + environment: + - TZ=Asia/Shanghai + - MYSQL_DATABASE=${MYSQL_DATABASE:-douyu_login} + - MYSQL_USER=${MYSQL_USER:-douyu_login} + - MYSQL_PASSWORD=${MYSQL_PASSWORD:?请在 .env 中设置 MYSQL_PASSWORD} + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:?请在 .env 中设置 MYSQL_ROOT_PASSWORD} + volumes: + - mysql-data:/var/lib/mysql + command: + - --character-set-server=utf8mb4 + - --collation-server=utf8mb4_0900_ai_ci + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h localhost -uroot -p$$MYSQL_ROOT_PASSWORD --silent"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 20s + +volumes: + mysql-data: diff --git a/pyproject.toml b/pyproject.toml index 5aefcf1..1847efa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "onnxruntime>=1.18.0", "scipy>=1.13.0", "pyexecjs>=1.5.1", + "pymysql>=1.1,<2", ] [build-system] diff --git a/scripts/migrate_sqlite_to_mysql.py b/scripts/migrate_sqlite_to_mysql.py new file mode 100644 index 0000000..30233f3 --- /dev/null +++ b/scripts/migrate_sqlite_to_mysql.py @@ -0,0 +1,192 @@ +"""将本项目 SQLite 数据安全迁移到 MySQL。""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from collections.abc import Iterable +from pathlib import Path + +from sqlalchemy import JSON, MetaData, create_engine, func, inspect, select +from sqlalchemy.engine import Connection, Engine +from sqlalchemy.schema import Table + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_SOURCE = PROJECT_ROOT / "data" / "web.db" +IGNORED_SOURCE_TABLES = {"alembic_version"} + + +def parse_args() -> argparse.Namespace: + """解析迁移参数。""" + parser = argparse.ArgumentParser(description="迁移 SQLite 数据到 MySQL") + parser.add_argument( + "--source", + type=Path, + default=DEFAULT_SOURCE, + help=f"SQLite 数据库路径，默认：{DEFAULT_SOURCE}", + ) + parser.add_argument( + "--target-url", + help="MySQL SQLAlchemy 连接串，例如 mysql+pymysql://user:password@host:3306/database", + ) + parser.add_argument( + "--batch-size", + type=int, + default=1000, + help="每批写入行数，默认 1000", + ) + args = parser.parse_args() + if args.batch_size < 1: + parser.error("--batch-size 必须大于 0") + return args + + +def _count_rows(connection: Connection, table: Table) -> int: + return connection.scalar(select(func.count()).select_from(table)) or 0 + + +def _chunks(rows: Iterable[dict], batch_size: int) -> Iterable[list[dict]]: + """将行数据分批，控制单次 SQL 的内存与事务体积。""" + batch: list[dict] = [] + for row in rows: + batch.append(row) + if len(batch) >= batch_size: + yield batch + batch = [] + if batch: + yield batch + + +def _normalize_value(value: object, target_column) -> object: + """将 SQLite 中以文本保存的 JSON 转回结构化值。""" + if value is None or not isinstance(target_column.type, JSON): + return value + if not isinstance(value, str): + return value + try: + return json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError(f"列 {target_column.name} 存在非法 JSON：{value[:120]!r}") from exc + + +def _source_rows(source_connection: Connection, source_table: Table, target_table: Table) -> Iterable[dict]: + columns = [column.name for column in target_table.columns] + missing_columns = [column for column in columns if column not in source_table.c] + if missing_columns: + raise RuntimeError( + f"源表 {source_table.name} 缺少列：{', '.join(missing_columns)}，" + "请先将 SQLite 升级到当前版本后再迁移" + ) + + primary_key_columns = list(source_table.primary_key.columns) + statement = select(*(source_table.c[name] for name in columns)) + if primary_key_columns: + statement = statement.order_by(*(column.asc() for column in primary_key_columns)) + + for row in source_connection.execute(statement).mappings(): + yield { + column_name: _normalize_value(row[column_name], target_table.c[column_name]) + for column_name in columns + } + + +def _ensure_target_is_empty(target_engine: Engine, target_tables: list[Table]) -> None: + """拒绝向已有业务数据的 MySQL 写入，防止误覆盖。""" + with target_engine.connect() as connection: + occupied = [ + table.name + for table in target_tables + if _count_rows(connection, table) > 0 + ] + if occupied: + raise RuntimeError(f"目标 MySQL 已存在业务数据，拒绝迁移：{', '.join(occupied)}") + + +def main() -> int: + """执行迁移、计数核验并返回进程退出码。""" + args = parse_args() + source_path = args.source.expanduser().resolve() + target_url = (args.target_url or "").strip() + + if not source_path.is_file(): + raise FileNotFoundError(f"未找到 SQLite 数据库：{source_path}") + + # 必须在导入数据库模块前设置，Alembic 环境才会使用迁移目标库。 + if target_url: + os.environ["DATABASE_URL"] = target_url + from web.backend import models # noqa: F401 + from web.backend.database import Base, DATABASE_URL, run_migrations + + target_url = target_url or DATABASE_URL + if not target_url.startswith("mysql+"): + raise ValueError("目标库必须是 MySQL；可传 --target-url，或设置 DB_HOST/DB_* 环境变量") + + print("正在初始化目标 MySQL 表结构...") + run_migrations() + + target_engine = create_engine(target_url, pool_pre_ping=True) + source_engine = create_engine(f"sqlite:///{source_path}") + target_tables = list(Base.metadata.sorted_tables) + target_table_names = {table.name for table in target_tables} + + try: + source_table_names = set(inspect(source_engine).get_table_names()) + unexpected_tables = source_table_names - target_table_names - IGNORED_SOURCE_TABLES + if unexpected_tables: + raise RuntimeError( + "源 SQLite 存在当前程序无法识别的表：" + f"{', '.join(sorted(unexpected_tables))}，为避免遗漏已停止迁移" + ) + + source_metadata = MetaData() + source_metadata.reflect( + bind=source_engine, + only=sorted(source_table_names & target_table_names), + ) + tables_to_copy = [ + table for table in target_tables if table.name in source_metadata.tables + ] + + _ensure_target_is_empty(target_engine, tables_to_copy) + + with source_engine.connect() as source_connection, target_engine.begin() as target_connection: + for target_table in tables_to_copy: + source_table = source_metadata.tables[target_table.name] + source_count = _count_rows(source_connection, source_table) + if not source_count: + print(f"{target_table.name}: 0 行，跳过") + continue + + print(f"{target_table.name}: 正在迁移 {source_count} 行...") + rows = _source_rows(source_connection, source_table, target_table) + for batch in _chunks(rows, args.batch_size): + target_connection.execute(target_table.insert(), batch) + + with source_engine.connect() as source_connection, target_engine.connect() as target_connection: + for target_table in tables_to_copy: + source_table = source_metadata.tables[target_table.name] + source_count = _count_rows(source_connection, source_table) + target_count = _count_rows(target_connection, target_table) + if source_count != target_count: + raise RuntimeError( + f"{target_table.name} 行数核验失败：" + f"SQLite={source_count}，MySQL={target_count}" + ) + print(f"{target_table.name}: 核验通过（{target_count} 行）") + finally: + source_engine.dispose() + target_engine.dispose() + + print("迁移完成，SQLite 源文件未修改。") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as exc: + print(f"迁移失败：{exc}", file=sys.stderr) + sys.exit(1) diff --git a/uv.lock b/uv.lock index 44b4c28..3cd1ec8 100644 --- a/uv.lock +++ b/uv.lock @@ -216,6 +216,7 @@ dependencies = [ { name = "pycryptodome" }, { name = "pydantic" }, { name = "pyexecjs" }, + { name = "pymysql" }, { name = "python-jose", extra = ["cryptography"] }, { name = "python-multipart" }, { name = "requests", extra = ["socks"] }, @@ -238,6 +239,7 @@ requires-dist = [ { name = "pycryptodome", specifier = ">=3.19.0" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "pyexecjs", specifier = ">=1.5.1" }, + { name = "pymysql", specifier = ">=1.1,<2" }, { name = "python-jose", extras = ["cryptography"], specifier = ">=3.3.0" }, { name = "python-multipart", specifier = ">=0.0.9" }, { name = "requests", extras = ["socks"], specifier = ">=2.31.0" }, @@ -565,6 +567,15 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/ba/8e/aedef81641c8dca6fd0fb7294de5bed9c45f3397d67fddf755c1042c2642/PyExecJS-1.5.1.tar.gz", hash = "sha256:34cc1d070976918183ff7bdc0ad71f8157a891c92708c00c5fbbff7a769f505c", size = 13344, upload-time = "2018-01-18T04:33:55.126Z" } +[[package]] +name = "pymysql" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/bc/1c6a92f385940f727daeecf3bacaf186e03875dff57197801046c583bcf0/pymysql-1.2.0.tar.gz", hash = "sha256:6c7b17ca686988104d7426c27895b455cdeea3e9d3ceb1270f0c3704fead8c33", size = 49021, upload-time = "2026-05-19T08:26:22.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/bd/2534e130295c8cfd4f0a2e31623baab7502278f1e97bcfe61db75656a77f/pymysql-1.2.0-py3-none-any.whl", hash = "sha256:62169ce6d5510f08e140c5e7990ee884a9764024e4a9a27b2cc11f1099322ae0", size = 45716, upload-time = "2026-05-19T08:26:20.974Z" }, +] + [[package]] name = "pysocks" version = "1.7.1" diff --git a/web/backend/database.py b/web/backend/database.py index bcf44c1..575b30b 100644 --- a/web/backend/database.py +++ b/web/backend/database.py @@ -2,13 +2,48 @@ import os from pathlib import Path +from urllib.parse import quote_plus + from sqlalchemy import create_engine, event from sqlalchemy.orm import sessionmaker, declarative_base PROJECT_ROOT = Path(__file__).resolve().parents[2] DB_PATH = PROJECT_ROOT / "data" / "web.db" DB_PATH.parent.mkdir(parents=True, exist_ok=True) -DATABASE_URL = os.getenv("DATABASE_URL", f"sqlite:///{DB_PATH}") + + +def _env_int(name: str, default: int) -> int: + """读取正整数环境变量，非法值回退到默认值。""" + try: + value = int(os.getenv(name, str(default))) + except ValueError: + return default + return value if value > 0 else default + + +def _get_database_url() -> str: + """优先使用完整连接串，否则按 DB_* 环境变量构造 MySQL 连接。""" + database_url = os.getenv("DATABASE_URL", "").strip() + if database_url: + return database_url + + db_host = os.getenv("DB_HOST", "").strip() + if db_host: + db_port = _env_int("DB_PORT", 3306) + db_name = os.getenv("DB_NAME", "douyu_login").strip() + db_user = os.getenv("DB_USER", "douyu_login").strip() + db_password = os.getenv("DB_PASSWORD", "") + if not db_name or not db_user or not db_password: + raise RuntimeError("使用 DB_HOST 时必须同时设置 DB_NAME、DB_USER 和 DB_PASSWORD") + return ( + f"mysql+pymysql://{quote_plus(db_user)}:{quote_plus(db_password)}" + f"@{db_host}:{db_port}/{db_name}?charset=utf8mb4" + ) + + return f"sqlite:///{DB_PATH}" + + +DATABASE_URL = _get_database_url() connect_args = ( {"check_same_thread": False, "timeout": 30} @@ -16,11 +51,20 @@ connect_args = ( else {} ) -engine = create_engine( - DATABASE_URL, - connect_args=connect_args, - echo=False, -) +engine_options = { + "connect_args": connect_args, + "echo": False, +} +if not DATABASE_URL.startswith("sqlite"): + engine_options.update( + pool_pre_ping=True, + pool_size=_env_int("DB_POOL_SIZE", 20), + max_overflow=_env_int("DB_MAX_OVERFLOW", 20), + pool_timeout=_env_int("DB_POOL_TIMEOUT", 30), + pool_recycle=_env_int("DB_POOL_RECYCLE", 1800), + ) + +engine = create_engine(DATABASE_URL, **engine_options) if DATABASE_URL.startswith("sqlite"): diff --git a/web/backend/migrations/versions/20260725_0009_database_performance_indexes.py b/web/backend/migrations/versions/20260725_0009_database_performance_indexes.py new file mode 100644 index 0000000..9c2341c --- /dev/null +++ b/web/backend/migrations/versions/20260725_0009_database_performance_indexes.py @@ -0,0 +1,67 @@ +"""补充高频查询组合索引 + +Revision ID: 20260725_0009 +Revises: 20260724_0008 +Create Date: 2026-07-25 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "20260725_0009" +down_revision: Union[str, None] = "20260724_0008" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _has_table(bind, table_name: str) -> bool: + return sa.inspect(bind).has_table(table_name) + + +def _indexes(bind, table_name: str) -> set[str]: + if not _has_table(bind, table_name): + return set() + return {index["name"] for index in sa.inspect(bind).get_indexes(table_name)} + + +def _create_index_if_missing(bind, name: str, table_name: str, columns: list[str]) -> None: + if name not in _indexes(bind, table_name): + op.create_index(name, table_name, columns) + + +def _drop_index_if_exists(bind, name: str, table_name: str) -> None: + if name in _indexes(bind, table_name): + op.drop_index(name, table_name=table_name) + + +INDEXES = [ + ("ix_accounts_assigned_to_id", "accounts", ["assigned_to", "id"]), + ("ix_accounts_tag_id", "accounts", ["tag", "id"]), + ("ix_login_tasks_account_status_id", "login_tasks", ["account_id", "status", "id"]), + ("ix_login_tasks_status_finished_at", "login_tasks", ["status", "finished_at"]), + ("ix_douyu_tasks_account_id", "douyu_tasks", ["account_id", "id"]), + ("ix_douyu_tasks_created_by_id", "douyu_tasks", ["created_by", "id"]), + ("ix_douyu_tasks_batch_status_id", "douyu_tasks", ["batch_id", "status", "id"]), + ("ix_huya_accounts_assigned_to_id", "huya_accounts", ["assigned_to", "id"]), + ("ix_huya_accounts_updated_at_id", "huya_accounts", ["updated_at", "id"]), + ("ix_huya_tasks_account_id", "huya_tasks", ["account_id", "id"]), + ("ix_huya_tasks_created_by_id", "huya_tasks", ["created_by", "id"]), + ("ix_huya_tasks_batch_status_id", "huya_tasks", ["batch_id", "status", "id"]), + ("ix_huya_register_items_batch_db_line", "huya_register_items", ["batch_db_id", "line"]), + ("ix_huya_register_success_logs_batch_id_id", "huya_register_success_logs", ["batch_id", "id"]), +] + + +def upgrade() -> None: + bind = op.get_bind() + for name, table_name, columns in INDEXES: + _create_index_if_missing(bind, name, table_name, columns) + + +def downgrade() -> None: + bind = op.get_bind() + for name, table_name, _ in reversed(INDEXES): + _drop_index_if_exists(bind, name, table_name)