From 3ce1c7a51bffebced7946d969b674837d0ad54c1 Mon Sep 17 00:00:00 2001 From: yml2213 Date: Sun, 30 Aug 2026 18:47:48 +0800 Subject: [PATCH] =?UTF-8?q?test:=20pytest=20=E5=B7=A5=E7=A8=8B=E5=8C=96?= =?UTF-8?q?=E8=90=BD=E5=9C=B0=20=E2=80=94=20=E6=9C=AC=E5=9C=B0=E9=97=A8?= =?UTF-8?q?=E7=A6=81=E3=80=81=E5=AE=B9=E5=99=A8=E6=B5=8B=E8=AF=95=E9=95=9C?= =?UTF-8?q?=E5=83=8F=E3=80=81=E8=BF=81=E7=A7=BB=E9=93=BE=E5=86=92=E7=83=9F?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 门禁: 本地 pre-push 钩子（.git/hooks，推送前强制跑全量 pytest） - Docker: 新增 test 镜像阶段（含 dev 依赖与 tests），compose 提供 --profile test run --rm test 入口；dev.sh 支持 ./dev.sh test - 测试基建: 根目录 conftest.py 统一 DATABASE_URL/APP_ENCRYPTION_KEY， 移除 9 个测试文件内的重复 setdefault（含多余 import os） - 迁移冒烟: tests/test_migrations.py 校验链线性、脚本可编译、空库整链 upgrade head 后与 Base.metadata 表/列/索引对齐 - 修复冒烟测试发现的漂移: YybRechargeTask.task_id 冗余 index=True （唯一索引已覆盖，迁移链未建普通索引，模型与真实 schema 对齐） - alembic.ini: path_separator=os 消除弃用告警；README 补测试章节 --- Dockerfile | 15 +++ README.md | 22 ++++ alembic.ini | 2 + conftest.py | 12 +++ dev.sh | 7 ++ docker-compose.yml | 12 +++ tests/test_account_sensitive_fields.py | 4 - tests/test_audit_logs.py | 4 - tests/test_cookie_custom_order.py | 3 - tests/test_cookie_operations.py | 4 - tests/test_douyu_gold_recharge_channel.py | 3 - tests/test_douyu_runner_proxy.py | 4 - tests/test_douyu_workbench_scopes.py | 4 - tests/test_huya_app_login.py | 3 - tests/test_migrations.py | 120 ++++++++++++++++++++++ tests/test_user_deletion.py | 4 - web/backend/models.py | 4 +- 17 files changed, 193 insertions(+), 34 deletions(-) create mode 100644 conftest.py create mode 100644 tests/test_migrations.py diff --git a/Dockerfile b/Dockerfile index cf8146b..0b0fe50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -99,3 +99,18 @@ HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ # 同一容器内启动 Web 与本机 Worker；入口脚本负责联动退出和信号转发。 ENTRYPOINT ["/app/docker-entrypoint.sh"] + +# ===== 阶段4: 测试镜像（含 dev 依赖与测试代码，可容器内跑 pytest）===== +# 用法: docker compose --profile test run --rm test +FROM runtime AS test + +ARG USE_CHINA_MIRRORS=true +COPY tests/ ./tests/ +COPY conftest.py ./ +RUN if [ "$USE_CHINA_MIRRORS" = "true" ]; then \ + (UV_INDEX_URL=https://mirrors.cloud.aliyuncs.com/pypi/simple/ uv sync --frozen --group dev || \ + UV_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/ uv sync --frozen --group dev || \ + uv sync --frozen --group dev); \ + else \ + uv sync --frozen --group dev; \ + fi diff --git a/README.md b/README.md index 6835565..5894086 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,28 @@ python -c "import secrets; print(secrets.token_urlsafe(32))" 3. IMAP 获取验证码 → 提交验证码 → 获取 Cookie ``` +## 运行测试 + +```bash +# 全量回归（SQLite 内存库，无需 MySQL / .env 配置） +uv run --group dev pytest # 或 ./dev.sh test + +# 只跑单个文件 / 用例 +uv run --group dev pytest tests/test_cookie_operations.py + +# 容器内回归（镜像内为构建时代码，改动后需重新构建） +docker compose --profile test run --rm test +``` + +测试使用各文件自建的 SQLite 内存引擎，互不干扰；根目录 `conftest.py` +统一兜底 `DATABASE_URL` / `APP_ENCRYPTION_KEY`，防止误连开发/生产库。 +数据库迁移链的冒烟校验见 `tests/test_migrations.py`（链线性 + 脚本可 +编译 + 空库整链 `upgrade head` 后与模型元数据表/列/索引对齐；MySQL +专属 DDL 由迁移脚本内的方言防护跳过，真实执行仍以部署流程 +`docker-entrypoint.sh` / `./deploy.sh` / `./dev.sh` 的 MySQL 为准）。 +已安装本地 pre-push 钩子：推送前自动跑全量测试，失败阻止推送 +（紧急绕过：`git push --no-verify` 或 `SKIP_TESTS=1 git push`）。 + ## 技术栈 - 后端：FastAPI + SQLAlchemy + SQLite + JWT diff --git a/alembic.ini b/alembic.ini index bed0014..6f1605f 100644 --- a/alembic.ini +++ b/alembic.ini @@ -1,6 +1,8 @@ [alembic] script_location = web/backend/migrations prepend_sys_path = . +# 迁移文件按 / 分隔的路径组织，避免旧式空白/冒号切分告警。 +path_separator = os sqlalchemy.url = sqlite:///data/web.db [loggers] diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..c0b6621 --- /dev/null +++ b/conftest.py @@ -0,0 +1,12 @@ +"""pytest 全局引导：在导入任何应用模块前固定测试数据库与加密密钥。 + +web/backend 的模块在 import 阶段就会读取 DATABASE_URL 创建全局引擎、 +读取 APP_ENCRYPTION_KEY 初始化敏感字段加密，因此这两个环境变量必须在 +测试模块加载之前设置好。各测试文件自身使用独立的 SQLite 内存引擎做 +数据隔离，这里统一兜底，避免误连开发/生产数据库。 +""" + +import os + +os.environ.setdefault("DATABASE_URL", "sqlite://") +os.environ.setdefault("APP_ENCRYPTION_KEY", "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") \ No newline at end of file diff --git a/dev.sh b/dev.sh index 8b47700..41a5edb 100755 --- a/dev.sh +++ b/dev.sh @@ -54,6 +54,13 @@ if [ "${1:-}" = "logs" ]; then exec tail -n "${LOG_TAIL_LINES:-200}" -F "$APP_LOG" fi +# 本地回归测试：仅依赖 uv，不需要 MySQL / .env 配置。 +# --group dev 确保 pytest 等 dev 依赖已安装。 +if [ "${1:-}" = "test" ]; then + shift + exec uv run --group dev pytest "$@" +fi + BACKEND_PID="" FRONTEND_PID="" WORKER_PID="" diff --git a/docker-compose.yml b/docker-compose.yml index 9876808..22d46f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,6 +75,18 @@ services: timeout: 5s retries: 3 + # 回归测试容器：docker compose --profile test run --rm test + # 镜像内为构建时的代码，改动代码后需重新构建（仅按需启动，不影响 docker compose up）。 + test: + build: + context: . + target: test + args: + USE_CHINA_MIRRORS: ${USE_CHINA_MIRRORS:-true} + profiles: ["test"] + container_name: douyu-login-test + entrypoint: ["python", "-m", "pytest"] + mysql: image: ${MYSQL_IMAGE:-docker.m.daocloud.io/library/mysql:8.4} container_name: douyu-login-db diff --git a/tests/test_account_sensitive_fields.py b/tests/test_account_sensitive_fields.py index 327408c..9f16389 100644 --- a/tests/test_account_sensitive_fields.py +++ b/tests/test_account_sensitive_fields.py @@ -1,9 +1,5 @@ -import os import unittest -os.environ.setdefault("DATABASE_URL", "sqlite://") -os.environ.setdefault("APP_ENCRYPTION_KEY", "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") - from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from fastapi import HTTPException diff --git a/tests/test_audit_logs.py b/tests/test_audit_logs.py index 5fa11a2..e3a0c10 100644 --- a/tests/test_audit_logs.py +++ b/tests/test_audit_logs.py @@ -1,12 +1,8 @@ """充值审计日志的权限、查询和脱敏测试。""" import json -import os import unittest -os.environ.setdefault("DATABASE_URL", "sqlite://") -os.environ.setdefault("APP_ENCRYPTION_KEY", "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") - from fastapi import HTTPException from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker diff --git a/tests/test_cookie_custom_order.py b/tests/test_cookie_custom_order.py index 5327f8b..58123bb 100644 --- a/tests/test_cookie_custom_order.py +++ b/tests/test_cookie_custom_order.py @@ -1,9 +1,6 @@ -import os import unittest from datetime import datetime, timedelta, timezone -os.environ.setdefault("DATABASE_URL", "sqlite://") - from sqlalchemy import create_engine from sqlalchemy.orm import Session diff --git a/tests/test_cookie_operations.py b/tests/test_cookie_operations.py index 4c7a7ba..d2a55fe 100644 --- a/tests/test_cookie_operations.py +++ b/tests/test_cookie_operations.py @@ -1,11 +1,7 @@ -import os import unittest from types import SimpleNamespace from unittest.mock import patch -os.environ.setdefault("DATABASE_URL", "sqlite://") -os.environ.setdefault("APP_ENCRYPTION_KEY", "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") - from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker diff --git a/tests/test_douyu_gold_recharge_channel.py b/tests/test_douyu_gold_recharge_channel.py index 9c81416..d06b87d 100644 --- a/tests/test_douyu_gold_recharge_channel.py +++ b/tests/test_douyu_gold_recharge_channel.py @@ -5,9 +5,6 @@ import os import unittest from unittest.mock import AsyncMock, Mock, patch -os.environ.setdefault("DATABASE_URL", "sqlite://") -os.environ.setdefault("APP_ENCRYPTION_KEY", "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") - from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker diff --git a/tests/test_douyu_runner_proxy.py b/tests/test_douyu_runner_proxy.py index 8910b13..bb1a1ea 100644 --- a/tests/test_douyu_runner_proxy.py +++ b/tests/test_douyu_runner_proxy.py @@ -1,12 +1,8 @@ """斗鱼任务代理接入三模式行为测试（静态 / API / 关闭）+ 写读分离。""" -import os import unittest from unittest.mock import Mock -os.environ.setdefault("DATABASE_URL", "sqlite://") -os.environ.setdefault("APP_ENCRYPTION_KEY", "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") - from core.douyu.activity_client import DouyuActivityClient from web.backend.services.douyu_runner import DouyuBatchRunner from web.backend.services.douyu_runner_core import DouyuBatchRunnerCore, DOUYU_PROXY_TASK_TYPES diff --git a/tests/test_douyu_workbench_scopes.py b/tests/test_douyu_workbench_scopes.py index edaf525..8f8021a 100644 --- a/tests/test_douyu_workbench_scopes.py +++ b/tests/test_douyu_workbench_scopes.py @@ -1,10 +1,6 @@ -import os import unittest from types import SimpleNamespace -os.environ.setdefault("DATABASE_URL", "sqlite://") -os.environ.setdefault("APP_ENCRYPTION_KEY", "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") - from sqlalchemy import create_engine, event from sqlalchemy.orm import sessionmaker diff --git a/tests/test_huya_app_login.py b/tests/test_huya_app_login.py index cbb7240..eef817c 100644 --- a/tests/test_huya_app_login.py +++ b/tests/test_huya_app_login.py @@ -6,9 +6,6 @@ import struct import unittest from unittest.mock import patch, MagicMock -os.environ.setdefault("DATABASE_URL", "sqlite://") -os.environ.setdefault("APP_ENCRYPTION_KEY", "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") - from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker diff --git a/tests/test_migrations.py b/tests/test_migrations.py new file mode 100644 index 0000000..c31c850 --- /dev/null +++ b/tests/test_migrations.py @@ -0,0 +1,120 @@ +"""Alembic 迁移链冒烟测试（不依赖真实 MySQL）。 + +覆盖三件事： +1. 链完整性：只有一个头版本、头与最新迁移一致、每个修订的 + down_revision 都能回溯到根（无悬挂引用、无分叉链）。 +2. 全部迁移脚本可编译：防止迁移文件被改坏后部署流程才暴露。 +3. 空库整链执行：迁移脚本自带方言防护（MySQL 专属 DDL 有 _is_mysql() + 之类分支），整条链可在 SQLite 上真实执行 `upgrade head`，随后校验 + 表/列/索引与 Base.metadata 模型元数据对齐（模型要求的不允许缺失）， + 并确认迁移新增的复合查询索引确实落库。 +""" + +import tempfile +import unittest +from pathlib import Path + +from alembic import command +from alembic.config import Config +from alembic.script import ScriptDirectory +from sqlalchemy import create_engine, inspect + +TESTS_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = TESTS_DIR.parent +VERSIONS_DIR = PROJECT_ROOT / "web" / "backend" / "migrations" / "versions" + +HEAD_REVISION = "20260830_0031" + +# 迁移新增、但不声明在模型里的复合查询索引。 +EXTRA_INDEXES = ( + "ix_douyu_tasks_handbook_scope_id", + "ix_douyu_tasks_handbook_scope_task_type_id", + "ix_douyu_tasks_status_finished_at_id", + "ix_login_tasks_status_finished_at_id", +) + + +def _migration_config() -> Config: + config = Config(str(PROJECT_ROOT / "alembic.ini")) + config.set_main_option( + "script_location", str(PROJECT_ROOT / "web" / "backend" / "migrations") + ) + return config + + +class MigrationSmokeTests(unittest.TestCase): + def test_chain_is_linear_and_head_matches_latest_file(self): + directory = ScriptDirectory.from_config(_migration_config()) + + heads = directory.get_heads() + self.assertEqual(len(heads), 1, f"迁移链出现多个头版本（存在分叉）: {heads}") + self.assertEqual(heads[0], HEAD_REVISION, f"头版本 {heads[0]} 与最新迁移不一致") + + version_files = sorted(path.name for path in VERSIONS_DIR.glob("*.py")) + revisions = list(directory.walk_revisions()) + revision_ids = {revision.revision for revision in revisions} + self.assertEqual( + len(revision_ids), + len(version_files), + "迁移文件数量与已加载修订数量不一致（存在无法加载的脚本）", + ) + for revision in revisions: + if revision.down_revision is not None: + self.assertIn( + revision.down_revision, + revision_ids, + f"{revision.revision} 引用了不存在的父版本 {revision.down_revision}", + ) + + def test_all_migration_scripts_compile(self): + for path in sorted(VERSIONS_DIR.glob("*.py")): + source = path.read_text(encoding="utf-8") + compile(source, str(path), "exec") # 语法/缩进等编译错误会在此抛出 + + def test_full_chain_applies_on_fresh_sqlite_and_matches_models(self): + """空库执行整条迁移链，校验表/列/索引与模型元数据对齐。""" + from web.backend.database import Base + import web.backend.database as database_module + + original_url = database_module.DATABASE_URL + with tempfile.TemporaryDirectory() as tmp: + db_path = Path(tmp) / "migrate.db" + try: + # env.py 会用模块级 DATABASE_URL 覆盖配置里的 sqlalchemy.url。 + database_module.DATABASE_URL = f"sqlite:///{db_path}" + command.upgrade(_migration_config(), "head") + + inspector = inspect(create_engine(f"sqlite:///{db_path}")) + db_tables = set(inspector.get_table_names()) + self.assertTrue(db_tables, "迁移链未创建任何表") + + model_tables = Base.metadata.tables + missing_tables = set(model_tables) - db_tables + self.assertFalse(missing_tables, f"模型要求但迁移未创建的表: {missing_tables}") + + all_db_indexes = set() + for name, table in model_tables.items(): + db_columns = {column["name"] for column in inspector.get_columns(name)} + model_columns = set(table.columns.keys()) + missing_columns = model_columns - db_columns + self.assertFalse(missing_columns, f"{name} 迁移后缺少列: {missing_columns}") + + db_indexes = { + index["name"] + for index in inspector.get_indexes(name) + if not index["name"].startswith("sqlite_autoindex") + } + all_db_indexes |= db_indexes + model_indexes = {index.name for index in table.indexes if index.name} + missing_indexes = model_indexes - db_indexes + self.assertFalse( + missing_indexes, f"{name} 迁移后缺少索引: {missing_indexes}" + ) + + # 迁移新增的复合查询索引（不入模型，但应存在）。 + for index_name in EXTRA_INDEXES: + self.assertIn( + index_name, all_db_indexes, f"迁移链未创建索引 {index_name}" + ) + finally: + database_module.DATABASE_URL = original_url \ No newline at end of file diff --git a/tests/test_user_deletion.py b/tests/test_user_deletion.py index 392823a..e207d1c 100644 --- a/tests/test_user_deletion.py +++ b/tests/test_user_deletion.py @@ -1,10 +1,6 @@ -import os import unittest from types import SimpleNamespace -os.environ.setdefault("DATABASE_URL", "sqlite://") -os.environ.setdefault("APP_ENCRYPTION_KEY", "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=") - from sqlalchemy import create_engine, event from sqlalchemy.orm import sessionmaker diff --git a/web/backend/models.py b/web/backend/models.py index 7ad5231..12dd9a4 100644 --- a/web/backend/models.py +++ b/web/backend/models.py @@ -168,7 +168,9 @@ class YybRechargeTask(Base): __tablename__ = "yyb_recharge_tasks" id = Column(Integer, primary_key=True, autoincrement=True) - task_id = Column(String(64), unique=True, nullable=False, index=True) + # 唯一索引（uq_yyb_recharge_tasks_task_id）已覆盖 task_id 查询，无需重复普通索引； + # 迁移 20260812_0020 也只建唯一索引，二者保持一致。 + task_id = Column(String(64), unique=True, nullable=False) created_by = Column(Integer, ForeignKey("users.id"), nullable=False, index=True) worker_job_id = Column(String(64), nullable=False, index=True) provider = Column(String(16), default="", nullable=False)