yml2213
c3fd299f3a
生产是 MySQL 8、测试是 SQLite，两者的方言差异在本地完全测不出来。今天为此 付出了两次代价： 1. `ORDER BY ck_checked_at ASC NULLS FIRST` 在 SQLite 上完全正常，本地 258 条 全绿，一部署到生产第一次巡检就 1064 语法错误（MySQL 不支持 NULLS FIRST）； 2. P1b 自动重登的结算复用调度器事务读取，在 SQLite 下看不出问题，但在 MySQL 的 REPEATABLE READ 下读到的是批次开始前的快照 —— 10 条实际成功的重登全部被记成 失败。SQLite 的读一致性与 MySQL 不同，这类问题跑一万遍也复现不出来。 改动： - 新增 tests/support/db.py：测试专用引擎/会话工具，连接串从 .env 读取， 默认指向独立的 douyu_login_test 库； - 新增 tests/conftest.py：会话级建好表结构，每个用例前后清空所有表做隔离； - 根 conftest.py 接管连接串设置：它在 pytest 收集阶段最先执行，在这里把 DATABASE_URL 指向测试库，应用模块首次 import 拿到就是 MySQL，不依赖任何 import 顺序（原先这里是 sqlite:// 兜底，改造后会变成「先建 SQLite engine 再打补丁换成 MySQL」，顺序一变就静默跑错库）； - 21 个测试文件的 setup 样板统一改用共享测试引擎； - 两个迁移测试改为在临时库 douyu_login_mig_<随机> 上跑完整 alembic upgrade head， 跑完 drop —— 迁移脚本本身也终于在 MySQL 上被验证； - dev.sh 与 README 的过时说明一并更新（原先写着「不需要 MySQL / .env 配置」； 「测试各自建 SQLite 内存库」）。 安全与易用： - 测试库名与开发库同名时**直接拒绝运行**（clear_tables 会 DELETE 所有表， 一旦 TEST_MYSQL_DATABASE 被指到 douyu_login 就是数据灾难）； - 连不上 MySQL 时给出可操作提示（先 ./dev.sh 起库 / 用 TEST_MYSQL_DATABASE 覆盖）， 而不是抛一段 SQLAlchemy 原始堆栈。 验收： - 全量 301 条通过，连跑两次结果一致（隔离有效），耗时约 12.5 秒（原 SQLite 约 3.6 秒）； - 单文件独立运行通过（不依赖全量执行顺序）； - `grep -rn sqlite tests/ conftest.py` 已无实际用法； - **方言确实生效**：临时把一条查询改回 `NULLS FIRST`，13 条测试立刻变红（含运行期 失败），还原即恢复；这条由 test_generated_sql_is_mysql_compatible 长期守着。 代价（已在 README 与规划附录 C.2 记明）：跑测试前必须先有 MySQL，pre-push 钩子 同理；CI 需要 service container。
409 lines
13 KiB
Bash
Executable File
409 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
# ===== 本地调试启动脚本：后端 reload + 前端热更新 =====
|
||
|
||
ROOT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
cd "$ROOT_DIR"
|
||
|
||
# 目录改名后，旧 shell 可能仍携带旧项目的 VIRTUAL_ENV。
|
||
# 开发脚本始终使用当前仓库的环境，不要求每次手工指定或重新激活环境。
|
||
unset VIRTUAL_ENV
|
||
export UV_PROJECT_ENVIRONMENT="$ROOT_DIR/.venv"
|
||
|
||
# 使用新的项目标识；可通过环境变量覆盖。
|
||
COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-live-hub-py}"
|
||
export COMPOSE_PROJECT_NAME
|
||
|
||
if [ -f "$ROOT_DIR/.env" ]; then
|
||
set -a
|
||
# 调试模式复用 Docker 部署的环境变量，尤其是 JWT 和敏感字段加密密钥。
|
||
# shellcheck disable=SC1091
|
||
source "$ROOT_DIR/.env"
|
||
set +a
|
||
fi
|
||
|
||
# 迁移期间优先复用旧项目卷，避免目录改名导致创建空数据库。
|
||
MYSQL_VOLUME_NAME="${MYSQL_VOLUME_NAME:-}"
|
||
MYSQL_VOLUME_EXTERNAL="${MYSQL_VOLUME_EXTERNAL:-false}"
|
||
if [ -z "$MYSQL_VOLUME_NAME" ] && command -v docker >/dev/null 2>&1 \
|
||
&& docker volume inspect douyu_login_py_mysql-data >/dev/null 2>&1; then
|
||
MYSQL_VOLUME_NAME="douyu_login_py_mysql-data"
|
||
MYSQL_VOLUME_EXTERNAL="true"
|
||
fi
|
||
export MYSQL_VOLUME_NAME
|
||
export MYSQL_VOLUME_EXTERNAL
|
||
|
||
BACKEND_HOST="${BACKEND_HOST:-0.0.0.0}"
|
||
# 8800 是项目默认开发端口并由脚本独占；其他显式端口发生冲突时只报错。
|
||
BACKEND_PORT_EXPLICIT=false
|
||
if [ -n "${BACKEND_PORT+x}" ]; then
|
||
BACKEND_PORT_EXPLICIT=true
|
||
fi
|
||
BACKEND_PORT="${BACKEND_PORT:-8800}"
|
||
FRONTEND_HOST="${FRONTEND_HOST:-0.0.0.0}"
|
||
FRONTEND_PORT="${FRONTEND_PORT:-5174}"
|
||
LOG_LEVEL="${LOG_LEVEL:-INFO}"
|
||
MYSQL_IMAGE="${MYSQL_IMAGE:-docker.m.daocloud.io/library/mysql:8.4}"
|
||
MYSQL_BIND_HOST="${MYSQL_BIND_HOST:-127.0.0.1}"
|
||
MYSQL_HOST_PORT="${MYSQL_HOST_PORT:-3307}"
|
||
MYSQL_DATABASE="${MYSQL_DATABASE:-douyu_login}"
|
||
MYSQL_USER="${MYSQL_USER:-douyu_login}"
|
||
DEV_DB_HOST="${DEV_DB_HOST:-127.0.0.1}"
|
||
DEV_DB_PORT="${DEV_DB_PORT:-$MYSQL_HOST_PORT}"
|
||
DB_HOST="$DEV_DB_HOST"
|
||
DB_PORT="$DEV_DB_PORT"
|
||
DB_NAME="${DB_NAME:-$MYSQL_DATABASE}"
|
||
DB_USER="${DB_USER:-$MYSQL_USER}"
|
||
DB_PASSWORD="${DB_PASSWORD:-${MYSQL_PASSWORD:-}}"
|
||
DB_POOL_SIZE="${DB_POOL_SIZE:-20}"
|
||
DB_MAX_OVERFLOW="${DB_MAX_OVERFLOW:-20}"
|
||
DB_POOL_TIMEOUT="${DB_POOL_TIMEOUT:-30}"
|
||
DB_POOL_RECYCLE="${DB_POOL_RECYCLE:-1800}"
|
||
YYB_WORKER_PORT="${YYB_WORKER_PORT:-8810}"
|
||
DEV_YYB_WORKER_URL="${DEV_YYB_WORKER_URL:-http://127.0.0.1:${YYB_WORKER_PORT}}"
|
||
YYB_WORKER_DATA_DIR="${YYB_WORKER_DATA_DIR:-$ROOT_DIR/data/yyb-worker-dev}"
|
||
YYB_WORKER_NODE_MODULES="$ROOT_DIR/services/yyb-worker/node_modules"
|
||
LOG_DIR="$ROOT_DIR/logs"
|
||
APP_LOG="$LOG_DIR/app-$(date '+%Y-%m-%d').log"
|
||
|
||
# 本地一键启动始终写入项目内的持久化日志目录，避免与旧的 /Users/.../app.log 混淆。
|
||
if [ "${1:-}" = "logs" ]; then
|
||
mkdir -p "$LOG_DIR"
|
||
echo "跟踪本地应用日志: ${APP_LOG}（Ctrl+C 退出）"
|
||
exec tail -n "${LOG_TAIL_LINES:-200}" -F "$APP_LOG"
|
||
fi
|
||
|
||
# 本地回归测试：现在跑在**真实 MySQL**上（不再用 SQLite）。
|
||
# 原因：生产是 MySQL，SQLite 与它有方言差异 —— 曾经出现「ORDER BY ... NULLS FIRST」
|
||
# 本地全绿、生产直接 1064 的事故（见 docs/自动化与资料收集规划.md 附录 C.2）。
|
||
# 因此需要：① 先 ./dev.sh 起 docker compose（MySQL 映射到 127.0.0.1:3307）；
|
||
# ② 测试库默认 douyu_login_test，可用 TEST_MYSQL_DATABASE 覆盖。
|
||
# 测试库与开发库同名时会被 tests/support/db.py 拒绝运行（防误清）。
|
||
# --group dev 确保 pytest 等 dev 依赖已安装。
|
||
if [ "${1:-}" = "test" ]; then
|
||
shift
|
||
exec uv run --group dev python -m pytest "$@"
|
||
fi
|
||
|
||
if [ "${1:-}" = "format" ]; then
|
||
shift
|
||
exec uv run --group dev python -m ruff format "$@"
|
||
fi
|
||
|
||
if [ "${1:-}" = "format-check" ]; then
|
||
shift
|
||
exec uv run --group dev python -m ruff format --check "$@"
|
||
fi
|
||
|
||
if [ "${1:-}" = "type-check" ]; then
|
||
shift
|
||
exec uv run --group dev python -m pyright "$@"
|
||
fi
|
||
|
||
# 与 Docker 镜像构建第一步对齐：tsc -b。
|
||
if [ "${1:-}" = "frontend-check" ]; then
|
||
exec "$ROOT_DIR/.githooks/run-frontend-tsc.sh"
|
||
fi
|
||
|
||
BACKEND_PID=""
|
||
FRONTEND_PID=""
|
||
WORKER_PID=""
|
||
|
||
stop_tree() {
|
||
local pid="$1"
|
||
local children
|
||
if [ -z "$pid" ] || ! kill -0 "$pid" 2>/dev/null; then
|
||
return
|
||
fi
|
||
children="$(pgrep -P "$pid" 2>/dev/null || true)"
|
||
for child in $children; do
|
||
stop_tree "$child"
|
||
done
|
||
kill "$pid" 2>/dev/null || true
|
||
}
|
||
|
||
cleanup() {
|
||
trap - EXIT INT TERM
|
||
stop_tree "$FRONTEND_PID"
|
||
stop_tree "$BACKEND_PID"
|
||
stop_tree "$WORKER_PID"
|
||
}
|
||
|
||
require_cmd() {
|
||
if ! command -v "$1" >/dev/null 2>&1; then
|
||
echo "缺少命令: $1"
|
||
exit 1
|
||
fi
|
||
}
|
||
|
||
require_cmd uv
|
||
require_cmd npm
|
||
require_cmd docker
|
||
|
||
detect_compose() {
|
||
if docker compose version >/dev/null 2>&1; then
|
||
echo "docker compose"
|
||
elif command -v docker-compose >/dev/null 2>&1; then
|
||
echo "docker-compose"
|
||
else
|
||
echo ""
|
||
fi
|
||
}
|
||
|
||
stop_legacy_containers() {
|
||
local name project
|
||
for name in douyu-login-db douyu-login; do
|
||
if ! docker inspect "$name" >/dev/null 2>&1; then
|
||
continue
|
||
fi
|
||
project="$(docker inspect "$name" --format '{{index .Config.Labels "com.docker.compose.project"}}' 2>/dev/null || true)"
|
||
if [ -n "$project" ] && [ "$project" != "$COMPOSE_PROJECT_NAME" ]; then
|
||
echo "停止旧 Compose 项目容器: $name (项目 $project)"
|
||
docker stop "$name" >/dev/null
|
||
fi
|
||
done
|
||
}
|
||
|
||
detect_backend_proxy_host() {
|
||
if [ "$BACKEND_HOST" != "0.0.0.0" ] && [ "$BACKEND_HOST" != "::" ]; then
|
||
echo "$BACKEND_HOST"
|
||
return
|
||
fi
|
||
# Vite 是本机进程，回环代理最稳定；LAN 客户端仍可经 Vite 的 0.0.0.0 监听访问。
|
||
echo "127.0.0.1"
|
||
}
|
||
|
||
port_in_use() {
|
||
command -v lsof >/dev/null 2>&1 && \
|
||
lsof -nP -iTCP:"$1" -sTCP:LISTEN >/dev/null 2>&1
|
||
}
|
||
|
||
show_port_owner() {
|
||
lsof -nP -iTCP:"$1" -sTCP:LISTEN 2>/dev/null | awk 'NR == 2 {print " 占用进程: " $1 " (pid " $2 ")"}'
|
||
}
|
||
|
||
ensure_backend_port_available() {
|
||
if ! port_in_use "$BACKEND_PORT"; then
|
||
return
|
||
fi
|
||
# 本地开发默认端口由本项目独占；编辑器残留的端口转发会让健康检查命中错误进程。
|
||
if [ "$BACKEND_PORT" = "8800" ]; then
|
||
local pids pid
|
||
pids="$(lsof -tiTCP:"$BACKEND_PORT" -sTCP:LISTEN 2>/dev/null || true)"
|
||
echo "默认后端端口 $BACKEND_PORT 已被占用，正在停止占用进程..."
|
||
show_port_owner "$BACKEND_PORT"
|
||
for pid in $pids; do
|
||
kill -TERM "$pid" 2>/dev/null || true
|
||
done
|
||
for _ in $(seq 1 5); do
|
||
if ! port_in_use "$BACKEND_PORT"; then
|
||
echo "端口 $BACKEND_PORT 已释放。"
|
||
return
|
||
fi
|
||
sleep 1
|
||
done
|
||
for pid in $pids; do
|
||
kill -KILL "$pid" 2>/dev/null || true
|
||
done
|
||
sleep 1
|
||
if ! port_in_use "$BACKEND_PORT"; then
|
||
echo "端口 $BACKEND_PORT 已释放。"
|
||
return
|
||
fi
|
||
echo "端口 $BACKEND_PORT 仍被占用，未启动后端。"
|
||
show_port_owner "$BACKEND_PORT"
|
||
exit 1
|
||
fi
|
||
|
||
if [ "$BACKEND_PORT_EXPLICIT" = true ]; then
|
||
echo "后端端口 $BACKEND_PORT 已被占用，无法启动。"
|
||
show_port_owner "$BACKEND_PORT"
|
||
echo "请关闭占用程序，或在 .env 中设置一个空闲的 BACKEND_PORT 后重试。"
|
||
exit 1
|
||
fi
|
||
echo "后端端口 $BACKEND_PORT 已被占用，无法启动。"
|
||
show_port_owner "$BACKEND_PORT"
|
||
echo "请在 .env 中设置空闲的 BACKEND_PORT 后重试。"
|
||
exit 1
|
||
}
|
||
|
||
if [ -n "${DATABASE_URL:-}" ] && [[ "$DATABASE_URL" != mysql+* ]]; then
|
||
echo "dev.sh 只支持 MySQL，请移除 SQLite DATABASE_URL: $DATABASE_URL"
|
||
exit 1
|
||
fi
|
||
|
||
if [ -z "${MYSQL_PASSWORD:-}" ] || [ -z "${MYSQL_ROOT_PASSWORD:-}" ]; then
|
||
echo "请先在 .env 中设置 MYSQL_PASSWORD 和 MYSQL_ROOT_PASSWORD"
|
||
exit 1
|
||
fi
|
||
|
||
if [ -z "$DB_PASSWORD" ]; then
|
||
echo "请先在 .env 中设置 DB_PASSWORD 或 MYSQL_PASSWORD"
|
||
exit 1
|
||
fi
|
||
|
||
if [ -z "${YYB_WORKER_KEY:-}" ]; then
|
||
echo "请先在 .env 中设置 YYB_WORKER_KEY"
|
||
exit 1
|
||
fi
|
||
|
||
ensure_backend_port_available
|
||
|
||
COMPOSE=$(detect_compose)
|
||
if [ -z "$COMPOSE" ]; then
|
||
echo "缺少 docker compose"
|
||
exit 1
|
||
fi
|
||
BACKEND_PROXY_HOST="${BACKEND_PROXY_HOST:-$(detect_backend_proxy_host)}"
|
||
BACKEND_PROXY_TARGET="${VITE_BACKEND_TARGET:-http://${BACKEND_PROXY_HOST}:${BACKEND_PORT}}"
|
||
|
||
mkdir -p data "$LOG_DIR"
|
||
mkdir -p "$YYB_WORKER_DATA_DIR"
|
||
|
||
trap cleanup EXIT INT TERM
|
||
|
||
start_yyb_worker() {
|
||
# Worker 跟随 dev.sh 生命周期：发现遗留进程(外部启动)时先停止，再自启，
|
||
# 确保运行的是当前代码且 Ctrl+C 时一并退出。
|
||
if curl -fsS "${DEV_YYB_WORKER_URL}/health" >/dev/null 2>&1; then
|
||
local leaked
|
||
leaked="$(lsof -ti tcp:"$YYB_WORKER_PORT" -sTCP:LISTEN 2>/dev/null | head -1 || true)"
|
||
if [ -n "$leaked" ]; then
|
||
echo "发现遗留 YYB Worker(pid $leaked)，正在停止并重新启动..."
|
||
kill "$leaked" 2>/dev/null || true
|
||
sleep 1
|
||
fi
|
||
fi
|
||
if [ ! -d "$YYB_WORKER_NODE_MODULES/jsdom" ]; then
|
||
echo "正在安装本地 YYB Worker Node 依赖..."
|
||
(cd "$ROOT_DIR/services/yyb-worker" && npm ci)
|
||
fi
|
||
|
||
echo "正在启动本地 YYB Worker..."
|
||
(
|
||
cd "$ROOT_DIR/services/yyb-worker/runtime"
|
||
YYB_WORKER_KEY="$YYB_WORKER_KEY" \
|
||
uv run --project "$ROOT_DIR" python scripts/yyb-worker.py \
|
||
--host 127.0.0.1 \
|
||
--port "$YYB_WORKER_PORT" \
|
||
--data-dir "$YYB_WORKER_DATA_DIR"
|
||
) &
|
||
WORKER_PID=$!
|
||
|
||
echo "等待本地 YYB Worker 就绪..."
|
||
for i in $(seq 1 15); do
|
||
if curl -fsS "${DEV_YYB_WORKER_URL}/health" >/dev/null 2>&1; then
|
||
echo "YYB Worker 已就绪"
|
||
return
|
||
fi
|
||
if [ "$i" = "15" ]; then
|
||
echo "YYB Worker 启动超时，查看日志："
|
||
echo "请查看当前终端中的 Worker 输出。"
|
||
exit 1
|
||
fi
|
||
sleep 1
|
||
done
|
||
}
|
||
|
||
echo "正在启动本地 MySQL..."
|
||
stop_legacy_containers
|
||
MYSQL_IMAGE="$MYSQL_IMAGE" \
|
||
MYSQL_BIND_HOST="$MYSQL_BIND_HOST" \
|
||
MYSQL_HOST_PORT="$MYSQL_HOST_PORT" \
|
||
$COMPOSE up -d mysql
|
||
|
||
echo "等待 MySQL 就绪..."
|
||
for i in $(seq 1 30); do
|
||
if $COMPOSE exec -T mysql sh -c 'mysqladmin ping -h localhost -uroot -p"$MYSQL_ROOT_PASSWORD" --silent' >/dev/null 2>&1; then
|
||
echo "MySQL 已就绪"
|
||
break
|
||
fi
|
||
if [ "$i" = "30" ]; then
|
||
echo "MySQL 启动超时，查看日志："
|
||
$COMPOSE logs --tail 50 mysql
|
||
exit 1
|
||
fi
|
||
sleep 2
|
||
done
|
||
|
||
echo "正在执行数据库迁移..."
|
||
(
|
||
export DB_HOST DB_PORT DB_NAME DB_USER DB_PASSWORD
|
||
uv run python -c 'from web.backend.database import run_migrations; run_migrations()'
|
||
)
|
||
echo "数据库迁移完成"
|
||
|
||
start_yyb_worker
|
||
|
||
echo ""
|
||
echo "=============================="
|
||
echo " 本地调试模式"
|
||
echo "=============================="
|
||
echo " 后端: http://${BACKEND_HOST}:${BACKEND_PORT}"
|
||
echo " 前端: http://localhost:${FRONTEND_PORT}"
|
||
echo " API 代理: ${BACKEND_PROXY_TARGET}"
|
||
echo " MySQL: ${DB_HOST}:${DB_PORT}/${DB_NAME}"
|
||
echo " YYB Worker: ${DEV_YYB_WORKER_URL}"
|
||
echo " 数据库迁移: 已检查至最新版本"
|
||
echo " 应用日志: ${APP_LOG}"
|
||
echo " 跟踪日志: ./dev.sh logs"
|
||
echo " 前端与 Worker 输出: 当前终端"
|
||
echo " 退出: Ctrl+C"
|
||
echo "=============================="
|
||
echo ""
|
||
|
||
(
|
||
export DB_HOST DB_PORT DB_NAME DB_USER DB_PASSWORD
|
||
export DB_POOL_SIZE DB_MAX_OVERFLOW DB_POOL_TIMEOUT DB_POOL_RECYCLE
|
||
export YYB_WORKER_KEY LOG_LEVEL LOG_DIR
|
||
export YYB_WORKER_URL="$DEV_YYB_WORKER_URL"
|
||
uv run python -m uvicorn web.backend.main:app \
|
||
--host "$BACKEND_HOST" \
|
||
--port "$BACKEND_PORT" \
|
||
--reload \
|
||
--reload-dir core \
|
||
--reload-dir utils \
|
||
--reload-dir web/backend
|
||
) &
|
||
BACKEND_PID=$!
|
||
|
||
echo "等待本地后端就绪..."
|
||
for i in $(seq 1 20); do
|
||
if curl -fsS --max-time 2 "http://127.0.0.1:${BACKEND_PORT}/api/health" 2>/dev/null | grep -q '"status":"ok"'; then
|
||
echo "后端已就绪，日志持续写入: $APP_LOG"
|
||
break
|
||
fi
|
||
if ! kill -0 "$BACKEND_PID" 2>/dev/null; then
|
||
echo "后端启动失败，最近应用日志："
|
||
tail -n 80 "$APP_LOG" 2>/dev/null || true
|
||
wait "$BACKEND_PID" || true
|
||
exit 1
|
||
fi
|
||
if [ "$i" = "20" ]; then
|
||
echo "后端启动超时，最近应用日志："
|
||
tail -n 80 "$APP_LOG" 2>/dev/null || true
|
||
exit 1
|
||
fi
|
||
sleep 1
|
||
done
|
||
|
||
(
|
||
cd web/frontend
|
||
VITE_BACKEND_TARGET="$BACKEND_PROXY_TARGET" \
|
||
npm run dev -- --host "$FRONTEND_HOST" --port "$FRONTEND_PORT"
|
||
) &
|
||
FRONTEND_PID=$!
|
||
|
||
while true; do
|
||
if ! kill -0 "$BACKEND_PID" 2>/dev/null; then
|
||
wait "$BACKEND_PID"
|
||
exit $?
|
||
fi
|
||
if ! kill -0 "$FRONTEND_PID" 2>/dev/null; then
|
||
wait "$FRONTEND_PID"
|
||
exit $?
|
||
fi
|
||
sleep 1
|
||
done
|