支持 MySQL 迁移与性能优化
This commit is contained in:
@@ -15,6 +15,24 @@ APP_ENCRYPTION_KEY=
|
|||||||
ADMIN_USERNAME=admin
|
ADMIN_USERNAME=admin
|
||||||
ADMIN_PASSWORD=admin123
|
ADMIN_PASSWORD=admin123
|
||||||
|
|
||||||
|
# Docker 对外端口(容器内仍监听 8800;服务器当前使用 8000)
|
||||||
|
APP_PORT=8000
|
||||||
|
|
||||||
|
# Docker 构建镜像源(服务器保持 true;本地测试可改为 false)
|
||||||
|
USE_CHINA_MIRRORS=true
|
||||||
|
|
||||||
|
# MySQL 配置(生产环境必须设置密码;MySQL 不映射宿主端口)
|
||||||
|
MYSQL_DATABASE=douyu_login
|
||||||
|
MYSQL_USER=douyu_login
|
||||||
|
MYSQL_PASSWORD=
|
||||||
|
MYSQL_ROOT_PASSWORD=
|
||||||
|
|
||||||
|
# 数据库连接池配置(仅 MySQL 生效)
|
||||||
|
DB_POOL_SIZE=20
|
||||||
|
DB_MAX_OVERFLOW=20
|
||||||
|
DB_POOL_TIMEOUT=30
|
||||||
|
DB_POOL_RECYCLE=1800
|
||||||
|
|
||||||
# Cookie 安全标志(生产环境 HTTPS 部署时设为 true)
|
# Cookie 安全标志(生产环境 HTTPS 部署时设为 true)
|
||||||
COOKIE_SECURE=false
|
COOKIE_SECURE=false
|
||||||
|
|
||||||
|
|||||||
+30
-15
@@ -13,38 +13,53 @@ RUN npm run build
|
|||||||
# ===== 阶段2: Python 运行时 =====
|
# ===== 阶段2: Python 运行时 =====
|
||||||
FROM python:3.12-slim AS runtime
|
FROM python:3.12-slim AS runtime
|
||||||
|
|
||||||
# 优先阿里云内网镜像,其次公网阿里云,最后官方源。
|
ARG USE_CHINA_MIRRORS=true
|
||||||
RUN cp /etc/apt/sources.list.d/debian.sources /tmp/debian.sources && \
|
|
||||||
sed -i 's/deb.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list.d/debian.sources && \
|
# 服务器默认使用阿里云镜像;本地构建可设 USE_CHINA_MIRRORS=false 直连官方源。
|
||||||
(apt-get update -o APT::Update::Error-Mode=any || \
|
RUN if [ "$USE_CHINA_MIRRORS" = "true" ]; then \
|
||||||
(sed -i 's/mirrors.cloud.aliyuncs.com/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources && \
|
cp /etc/apt/sources.list.d/debian.sources /tmp/debian.sources && \
|
||||||
apt-get update -o APT::Update::Error-Mode=any) || \
|
sed -i 's/deb.debian.org/mirrors.cloud.aliyuncs.com/g' /etc/apt/sources.list.d/debian.sources && \
|
||||||
(cp /tmp/debian.sources /etc/apt/sources.list.d/debian.sources && \
|
(apt-get update -o APT::Update::Error-Mode=any || \
|
||||||
apt-get update -o APT::Update::Error-Mode=any)) && \
|
(sed -i 's/mirrors.cloud.aliyuncs.com/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources && \
|
||||||
|
apt-get update -o APT::Update::Error-Mode=any) || \
|
||||||
|
(cp /tmp/debian.sources /etc/apt/sources.list.d/debian.sources && \
|
||||||
|
apt-get update -o APT::Update::Error-Mode=any)); \
|
||||||
|
else \
|
||||||
|
apt-get update -o APT::Update::Error-Mode=any; \
|
||||||
|
fi && \
|
||||||
apt-get install -y --no-install-recommends \
|
apt-get install -y --no-install-recommends \
|
||||||
libgl1 \
|
libgl1 \
|
||||||
libglib2.0-0 \
|
libglib2.0-0 \
|
||||||
nodejs \
|
nodejs \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# 安装 uv:优先阿里云内网 PyPI → 公网阿里云 → 官方。
|
# 安装 uv:服务器走阿里云镜像,本地直连官方 PyPI。
|
||||||
RUN pip install -i https://mirrors.cloud.aliyuncs.com/pypi/simple/ uv || \
|
RUN if [ "$USE_CHINA_MIRRORS" = "true" ]; then \
|
||||||
pip install -i https://mirrors.aliyun.com/pypi/simple/ uv || \
|
pip install -i https://mirrors.cloud.aliyuncs.com/pypi/simple/ uv || \
|
||||||
pip install uv
|
pip install -i https://mirrors.aliyun.com/pypi/simple/ uv || \
|
||||||
|
pip install uv; \
|
||||||
|
else \
|
||||||
|
pip install uv; \
|
||||||
|
fi
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# 先复制依赖文件,利用 Docker 缓存层
|
# 先复制依赖文件,利用 Docker 缓存层
|
||||||
COPY pyproject.toml uv.lock* README.md alembic.ini ./
|
COPY pyproject.toml uv.lock* README.md alembic.ini ./
|
||||||
RUN uv venv /app/.venv && \
|
RUN uv venv /app/.venv && \
|
||||||
(UV_INDEX_URL=https://mirrors.cloud.aliyuncs.com/pypi/simple/ uv pip install -e . || \
|
if [ "$USE_CHINA_MIRRORS" = "true" ]; then \
|
||||||
UV_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/ uv pip install -e . || \
|
(UV_INDEX_URL=https://mirrors.cloud.aliyuncs.com/pypi/simple/ uv pip install -e . || \
|
||||||
uv pip install -e .)
|
UV_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/ uv pip install -e . || \
|
||||||
|
uv pip install -e .); \
|
||||||
|
else \
|
||||||
|
uv pip install -e .; \
|
||||||
|
fi
|
||||||
|
|
||||||
# 复制项目代码
|
# 复制项目代码
|
||||||
COPY core/ ./core/
|
COPY core/ ./core/
|
||||||
COPY utils/ ./utils/
|
COPY utils/ ./utils/
|
||||||
COPY web/backend/ ./web/backend/
|
COPY web/backend/ ./web/backend/
|
||||||
|
COPY scripts/ ./scripts/
|
||||||
|
|
||||||
# 复制前端构建产物
|
# 复制前端构建产物
|
||||||
COPY --from=frontend-builder /build/dist ./web/frontend/dist
|
COPY --from=frontend-builder /build/dist ./web/frontend/dist
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ set -e
|
|||||||
|
|
||||||
ROOT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
ROOT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
|
APP_PORT="${APP_PORT:-8000}"
|
||||||
|
|
||||||
# ── 工具函数 ──────────────────────────────────────────────
|
# ── 工具函数 ──────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -44,9 +45,6 @@ cmd_deploy() {
|
|||||||
# 创建数据和日志目录
|
# 创建数据和日志目录
|
||||||
mkdir -p data logs
|
mkdir -p data logs
|
||||||
|
|
||||||
# 先停掉旧容器(避免名称冲突)
|
|
||||||
$COMPOSE down 2>/dev/null || true
|
|
||||||
|
|
||||||
# 构建并启动
|
# 构建并启动
|
||||||
echo "正在构建镜像(首次构建约3-5分钟)..."
|
echo "正在构建镜像(首次构建约3-5分钟)..."
|
||||||
$COMPOSE build
|
$COMPOSE build
|
||||||
@@ -60,12 +58,12 @@ cmd_deploy() {
|
|||||||
|
|
||||||
# 健康检查
|
# 健康检查
|
||||||
for i in $(seq 1 15); do
|
for i in $(seq 1 15); do
|
||||||
if curl -s http://localhost:8800/api/health | grep -q "ok" 2>/dev/null; then
|
if curl -s "http://localhost:${APP_PORT}/api/health" | grep -q "ok" 2>/dev/null; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "=============================="
|
echo "=============================="
|
||||||
echo " ✅ 部署成功!"
|
echo " ✅ 部署成功!"
|
||||||
echo "=============================="
|
echo "=============================="
|
||||||
echo " 访问地址: http://localhost:8800"
|
echo " 访问地址: http://localhost:${APP_PORT}"
|
||||||
echo " 默认账号见环境变量 ADMIN_USERNAME/ADMIN_PASSWORD(未设置则为 admin/admin123)"
|
echo " 默认账号见环境变量 ADMIN_USERNAME/ADMIN_PASSWORD(未设置则为 admin/admin123)"
|
||||||
echo ""
|
echo ""
|
||||||
echo " 查看日志: $COMPOSE logs -f"
|
echo " 查看日志: $COMPOSE logs -f"
|
||||||
@@ -92,7 +90,7 @@ cmd_reset() {
|
|||||||
echo "=============================="
|
echo "=============================="
|
||||||
echo ""
|
echo ""
|
||||||
echo " 将删除以下内容:"
|
echo " 将删除以下内容:"
|
||||||
echo " • 数据库文件 (data/web.db)"
|
echo " • MySQL 数据卷"
|
||||||
echo " • 历史Cookie (data/cookies/)"
|
echo " • 历史Cookie (data/cookies/)"
|
||||||
echo " • 应用日志 (logs/)"
|
echo " • 应用日志 (logs/)"
|
||||||
echo " • Docker容器和镜像"
|
echo " • Docker容器和镜像"
|
||||||
@@ -110,11 +108,11 @@ cmd_reset() {
|
|||||||
# 停止并删除容器
|
# 停止并删除容器
|
||||||
if [ -n "$COMPOSE" ]; then
|
if [ -n "$COMPOSE" ]; then
|
||||||
echo " 停止并删除容器..."
|
echo " 停止并删除容器..."
|
||||||
$COMPOSE down --rmi local 2>/dev/null || $COMPOSE down 2>/dev/null || true
|
$COMPOSE down --volumes --rmi local 2>/dev/null || $COMPOSE down --volumes 2>/dev/null || true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 删除数据文件
|
# 删除数据文件
|
||||||
echo " 删除数据库..."
|
echo " 删除旧 SQLite 数据..."
|
||||||
rm -rf data/web.db data/web.db-shm data/web.db-wal
|
rm -rf data/web.db data/web.db-shm data/web.db-wal
|
||||||
|
|
||||||
echo " 删除Cookie..."
|
echo " 删除Cookie..."
|
||||||
@@ -143,6 +141,42 @@ cmd_logs() {
|
|||||||
$COMPOSE logs -f
|
$COMPOSE logs -f
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cmd_migrate_mysql() {
|
||||||
|
COMPOSE=$(detect_compose)
|
||||||
|
if [ -z "$COMPOSE" ]; then
|
||||||
|
echo "❌ 未检测到 docker compose"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ ! -f data/web.db ]; then
|
||||||
|
echo "❌ 未找到 data/web.db,无法迁移 SQLite 数据"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "正在启动 MySQL..."
|
||||||
|
$COMPOSE up -d mysql
|
||||||
|
|
||||||
|
echo "等待 MySQL 就绪..."
|
||||||
|
for i in $(seq 1 30); do
|
||||||
|
if $COMPOSE exec -T mysql sh -c 'mysqladmin ping -h localhost -uroot -p"$MYSQL_ROOT_PASSWORD" --silent' >/dev/null 2>&1; then
|
||||||
|
echo "MySQL 已就绪"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
if [ "$i" = "30" ]; then
|
||||||
|
echo "❌ MySQL 启动超时,查看日志:"
|
||||||
|
$COMPOSE logs --tail 50 mysql
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "正在构建迁移运行镜像..."
|
||||||
|
$COMPOSE build douyu-login
|
||||||
|
|
||||||
|
echo "开始迁移 SQLite 到 MySQL..."
|
||||||
|
$COMPOSE run --rm --no-deps douyu-login \
|
||||||
|
python scripts/migrate_sqlite_to_mysql.py --source /app/data/web.db
|
||||||
|
}
|
||||||
|
|
||||||
cmd_stop() {
|
cmd_stop() {
|
||||||
COMPOSE=$(detect_compose)
|
COMPOSE=$(detect_compose)
|
||||||
if [ -z "$COMPOSE" ]; then
|
if [ -z "$COMPOSE" ]; then
|
||||||
@@ -180,13 +214,14 @@ cmd_help() {
|
|||||||
echo " 命令:"
|
echo " 命令:"
|
||||||
echo " (无参数) 构建并部署"
|
echo " (无参数) 构建并部署"
|
||||||
echo " reset 删除所有数据并重置(需确认)"
|
echo " reset 删除所有数据并重置(需确认)"
|
||||||
|
echo " migrate-mysql 先启动 MySQL 并迁移 data/web.db"
|
||||||
echo " logs 查看实时日志"
|
echo " logs 查看实时日志"
|
||||||
echo " stop 停止服务"
|
echo " stop 停止服务"
|
||||||
echo " restart 重启服务"
|
echo " restart 重启服务"
|
||||||
echo " dev 本地调试启动(后端 reload + 前端热更新)"
|
echo " dev 本地调试启动(后端 reload + 前端热更新)"
|
||||||
echo " help 显示帮助"
|
echo " help 显示帮助"
|
||||||
echo ""
|
echo ""
|
||||||
echo " 访问地址: http://localhost:8800"
|
echo " 访问地址: http://localhost:${APP_PORT}"
|
||||||
echo " 调试地址: http://localhost:5174"
|
echo " 调试地址: http://localhost:5174"
|
||||||
echo " 默认账号见环境变量 ADMIN_USERNAME/ADMIN_PASSWORD"
|
echo " 默认账号见环境变量 ADMIN_USERNAME/ADMIN_PASSWORD"
|
||||||
echo ""
|
echo ""
|
||||||
@@ -196,6 +231,7 @@ cmd_help() {
|
|||||||
|
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
reset) cmd_reset ;;
|
reset) cmd_reset ;;
|
||||||
|
migrate-mysql) cmd_migrate_mysql ;;
|
||||||
logs) cmd_logs ;;
|
logs) cmd_logs ;;
|
||||||
stop) cmd_stop ;;
|
stop) cmd_stop ;;
|
||||||
restart) cmd_restart ;;
|
restart) cmd_restart ;;
|
||||||
|
|||||||
+46
-3
@@ -1,16 +1,31 @@
|
|||||||
services:
|
services:
|
||||||
douyu-login:
|
douyu-login:
|
||||||
build: .
|
build:
|
||||||
|
context: .
|
||||||
|
args:
|
||||||
|
# 服务器默认走阿里云镜像;本地测试设为 false。
|
||||||
|
USE_CHINA_MIRRORS: ${USE_CHINA_MIRRORS:-true}
|
||||||
container_name: douyu-login
|
container_name: douyu-login
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
ports:
|
ports:
|
||||||
- "8800:8800"
|
# 保持现有服务端口;可通过 APP_PORT 覆盖。
|
||||||
|
- "${APP_PORT:-8000}:8800"
|
||||||
volumes:
|
volumes:
|
||||||
# 持久化数据库和日志
|
# 持久化运行时文件和日志,数据库由 mysql-data 卷管理。
|
||||||
- ./data:/app/data
|
- ./data:/app/data
|
||||||
- ./logs:/app/logs
|
- ./logs:/app/logs
|
||||||
environment:
|
environment:
|
||||||
- TZ=Asia/Shanghai
|
- TZ=Asia/Shanghai
|
||||||
|
# MySQL 仅在 Compose 内网开放,不映射宿主机端口。
|
||||||
|
- DB_HOST=mysql
|
||||||
|
- DB_PORT=3306
|
||||||
|
- DB_NAME=${MYSQL_DATABASE:-douyu_login}
|
||||||
|
- DB_USER=${MYSQL_USER:-douyu_login}
|
||||||
|
- DB_PASSWORD=${MYSQL_PASSWORD:?请在 .env 中设置 MYSQL_PASSWORD}
|
||||||
|
- DB_POOL_SIZE=${DB_POOL_SIZE:-20}
|
||||||
|
- DB_MAX_OVERFLOW=${DB_MAX_OVERFLOW:-20}
|
||||||
|
- DB_POOL_TIMEOUT=${DB_POOL_TIMEOUT:-30}
|
||||||
|
- DB_POOL_RECYCLE=${DB_POOL_RECYCLE:-1800}
|
||||||
# JWT 密钥(生产环境务必修改,可用 python -c "import secrets; print(secrets.token_urlsafe(32))" 生成)
|
# JWT 密钥(生产环境务必修改,可用 python -c "import secrets; print(secrets.token_urlsafe(32))" 生成)
|
||||||
- JWT_SECRET_KEY=${JWT_SECRET_KEY:-}
|
- JWT_SECRET_KEY=${JWT_SECRET_KEY:-}
|
||||||
# 敏感字段落盘加密密钥(生产环境务必长期保持不变)
|
# 敏感字段落盘加密密钥(生产环境务必长期保持不变)
|
||||||
@@ -26,6 +41,9 @@ services:
|
|||||||
- MAIL_ROUNDCUBE_URL=${MAIL_ROUNDCUBE_URL:-}
|
- MAIL_ROUNDCUBE_URL=${MAIL_ROUNDCUBE_URL:-}
|
||||||
# Uvicorn reload(生产环境保持 false)
|
# Uvicorn reload(生产环境保持 false)
|
||||||
- UVICORN_RELOAD=${UVICORN_RELOAD:-false}
|
- UVICORN_RELOAD=${UVICORN_RELOAD:-false}
|
||||||
|
depends_on:
|
||||||
|
mysql:
|
||||||
|
condition: service_healthy
|
||||||
# 增加文件描述符限制,避免 "Too many open files" 错误
|
# 增加文件描述符限制,避免 "Too many open files" 错误
|
||||||
ulimits:
|
ulimits:
|
||||||
nofile:
|
nofile:
|
||||||
@@ -36,3 +54,28 @@ services:
|
|||||||
interval: 30s
|
interval: 30s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 3
|
retries: 3
|
||||||
|
|
||||||
|
mysql:
|
||||||
|
image: docker.m.daocloud.io/library/mysql:8.4
|
||||||
|
container_name: douyu-login-db
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
- TZ=Asia/Shanghai
|
||||||
|
- MYSQL_DATABASE=${MYSQL_DATABASE:-douyu_login}
|
||||||
|
- MYSQL_USER=${MYSQL_USER:-douyu_login}
|
||||||
|
- MYSQL_PASSWORD=${MYSQL_PASSWORD:?请在 .env 中设置 MYSQL_PASSWORD}
|
||||||
|
- MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:?请在 .env 中设置 MYSQL_ROOT_PASSWORD}
|
||||||
|
volumes:
|
||||||
|
- mysql-data:/var/lib/mysql
|
||||||
|
command:
|
||||||
|
- --character-set-server=utf8mb4
|
||||||
|
- --collation-server=utf8mb4_0900_ai_ci
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "mysqladmin ping -h localhost -uroot -p$$MYSQL_ROOT_PASSWORD --silent"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 12
|
||||||
|
start_period: 20s
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
mysql-data:
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ dependencies = [
|
|||||||
"onnxruntime>=1.18.0",
|
"onnxruntime>=1.18.0",
|
||||||
"scipy>=1.13.0",
|
"scipy>=1.13.0",
|
||||||
"pyexecjs>=1.5.1",
|
"pyexecjs>=1.5.1",
|
||||||
|
"pymysql>=1.1,<2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
|
|||||||
@@ -0,0 +1,192 @@
|
|||||||
|
"""将本项目 SQLite 数据安全迁移到 MySQL。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from collections.abc import Iterable
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import JSON, MetaData, create_engine, func, inspect, select
|
||||||
|
from sqlalchemy.engine import Connection, Engine
|
||||||
|
from sqlalchemy.schema import Table
|
||||||
|
|
||||||
|
|
||||||
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
DEFAULT_SOURCE = PROJECT_ROOT / "data" / "web.db"
|
||||||
|
IGNORED_SOURCE_TABLES = {"alembic_version"}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
"""解析迁移参数。"""
|
||||||
|
parser = argparse.ArgumentParser(description="迁移 SQLite 数据到 MySQL")
|
||||||
|
parser.add_argument(
|
||||||
|
"--source",
|
||||||
|
type=Path,
|
||||||
|
default=DEFAULT_SOURCE,
|
||||||
|
help=f"SQLite 数据库路径,默认:{DEFAULT_SOURCE}",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--target-url",
|
||||||
|
help="MySQL SQLAlchemy 连接串,例如 mysql+pymysql://user:password@host:3306/database",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--batch-size",
|
||||||
|
type=int,
|
||||||
|
default=1000,
|
||||||
|
help="每批写入行数,默认 1000",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.batch_size < 1:
|
||||||
|
parser.error("--batch-size 必须大于 0")
|
||||||
|
return args
|
||||||
|
|
||||||
|
|
||||||
|
def _count_rows(connection: Connection, table: Table) -> int:
|
||||||
|
return connection.scalar(select(func.count()).select_from(table)) or 0
|
||||||
|
|
||||||
|
|
||||||
|
def _chunks(rows: Iterable[dict], batch_size: int) -> Iterable[list[dict]]:
|
||||||
|
"""将行数据分批,控制单次 SQL 的内存与事务体积。"""
|
||||||
|
batch: list[dict] = []
|
||||||
|
for row in rows:
|
||||||
|
batch.append(row)
|
||||||
|
if len(batch) >= batch_size:
|
||||||
|
yield batch
|
||||||
|
batch = []
|
||||||
|
if batch:
|
||||||
|
yield batch
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_value(value: object, target_column) -> object:
|
||||||
|
"""将 SQLite 中以文本保存的 JSON 转回结构化值。"""
|
||||||
|
if value is None or not isinstance(target_column.type, JSON):
|
||||||
|
return value
|
||||||
|
if not isinstance(value, str):
|
||||||
|
return value
|
||||||
|
try:
|
||||||
|
return json.loads(value)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise ValueError(f"列 {target_column.name} 存在非法 JSON:{value[:120]!r}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _source_rows(source_connection: Connection, source_table: Table, target_table: Table) -> Iterable[dict]:
|
||||||
|
columns = [column.name for column in target_table.columns]
|
||||||
|
missing_columns = [column for column in columns if column not in source_table.c]
|
||||||
|
if missing_columns:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"源表 {source_table.name} 缺少列:{', '.join(missing_columns)},"
|
||||||
|
"请先将 SQLite 升级到当前版本后再迁移"
|
||||||
|
)
|
||||||
|
|
||||||
|
primary_key_columns = list(source_table.primary_key.columns)
|
||||||
|
statement = select(*(source_table.c[name] for name in columns))
|
||||||
|
if primary_key_columns:
|
||||||
|
statement = statement.order_by(*(column.asc() for column in primary_key_columns))
|
||||||
|
|
||||||
|
for row in source_connection.execute(statement).mappings():
|
||||||
|
yield {
|
||||||
|
column_name: _normalize_value(row[column_name], target_table.c[column_name])
|
||||||
|
for column_name in columns
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_target_is_empty(target_engine: Engine, target_tables: list[Table]) -> None:
|
||||||
|
"""拒绝向已有业务数据的 MySQL 写入,防止误覆盖。"""
|
||||||
|
with target_engine.connect() as connection:
|
||||||
|
occupied = [
|
||||||
|
table.name
|
||||||
|
for table in target_tables
|
||||||
|
if _count_rows(connection, table) > 0
|
||||||
|
]
|
||||||
|
if occupied:
|
||||||
|
raise RuntimeError(f"目标 MySQL 已存在业务数据,拒绝迁移:{', '.join(occupied)}")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
"""执行迁移、计数核验并返回进程退出码。"""
|
||||||
|
args = parse_args()
|
||||||
|
source_path = args.source.expanduser().resolve()
|
||||||
|
target_url = (args.target_url or "").strip()
|
||||||
|
|
||||||
|
if not source_path.is_file():
|
||||||
|
raise FileNotFoundError(f"未找到 SQLite 数据库:{source_path}")
|
||||||
|
|
||||||
|
# 必须在导入数据库模块前设置,Alembic 环境才会使用迁移目标库。
|
||||||
|
if target_url:
|
||||||
|
os.environ["DATABASE_URL"] = target_url
|
||||||
|
from web.backend import models # noqa: F401
|
||||||
|
from web.backend.database import Base, DATABASE_URL, run_migrations
|
||||||
|
|
||||||
|
target_url = target_url or DATABASE_URL
|
||||||
|
if not target_url.startswith("mysql+"):
|
||||||
|
raise ValueError("目标库必须是 MySQL;可传 --target-url,或设置 DB_HOST/DB_* 环境变量")
|
||||||
|
|
||||||
|
print("正在初始化目标 MySQL 表结构...")
|
||||||
|
run_migrations()
|
||||||
|
|
||||||
|
target_engine = create_engine(target_url, pool_pre_ping=True)
|
||||||
|
source_engine = create_engine(f"sqlite:///{source_path}")
|
||||||
|
target_tables = list(Base.metadata.sorted_tables)
|
||||||
|
target_table_names = {table.name for table in target_tables}
|
||||||
|
|
||||||
|
try:
|
||||||
|
source_table_names = set(inspect(source_engine).get_table_names())
|
||||||
|
unexpected_tables = source_table_names - target_table_names - IGNORED_SOURCE_TABLES
|
||||||
|
if unexpected_tables:
|
||||||
|
raise RuntimeError(
|
||||||
|
"源 SQLite 存在当前程序无法识别的表:"
|
||||||
|
f"{', '.join(sorted(unexpected_tables))},为避免遗漏已停止迁移"
|
||||||
|
)
|
||||||
|
|
||||||
|
source_metadata = MetaData()
|
||||||
|
source_metadata.reflect(
|
||||||
|
bind=source_engine,
|
||||||
|
only=sorted(source_table_names & target_table_names),
|
||||||
|
)
|
||||||
|
tables_to_copy = [
|
||||||
|
table for table in target_tables if table.name in source_metadata.tables
|
||||||
|
]
|
||||||
|
|
||||||
|
_ensure_target_is_empty(target_engine, tables_to_copy)
|
||||||
|
|
||||||
|
with source_engine.connect() as source_connection, target_engine.begin() as target_connection:
|
||||||
|
for target_table in tables_to_copy:
|
||||||
|
source_table = source_metadata.tables[target_table.name]
|
||||||
|
source_count = _count_rows(source_connection, source_table)
|
||||||
|
if not source_count:
|
||||||
|
print(f"{target_table.name}: 0 行,跳过")
|
||||||
|
continue
|
||||||
|
|
||||||
|
print(f"{target_table.name}: 正在迁移 {source_count} 行...")
|
||||||
|
rows = _source_rows(source_connection, source_table, target_table)
|
||||||
|
for batch in _chunks(rows, args.batch_size):
|
||||||
|
target_connection.execute(target_table.insert(), batch)
|
||||||
|
|
||||||
|
with source_engine.connect() as source_connection, target_engine.connect() as target_connection:
|
||||||
|
for target_table in tables_to_copy:
|
||||||
|
source_table = source_metadata.tables[target_table.name]
|
||||||
|
source_count = _count_rows(source_connection, source_table)
|
||||||
|
target_count = _count_rows(target_connection, target_table)
|
||||||
|
if source_count != target_count:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"{target_table.name} 行数核验失败:"
|
||||||
|
f"SQLite={source_count},MySQL={target_count}"
|
||||||
|
)
|
||||||
|
print(f"{target_table.name}: 核验通过({target_count} 行)")
|
||||||
|
finally:
|
||||||
|
source_engine.dispose()
|
||||||
|
target_engine.dispose()
|
||||||
|
|
||||||
|
print("迁移完成,SQLite 源文件未修改。")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
sys.exit(main())
|
||||||
|
except Exception as exc:
|
||||||
|
print(f"迁移失败:{exc}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
@@ -216,6 +216,7 @@ dependencies = [
|
|||||||
{ name = "pycryptodome" },
|
{ name = "pycryptodome" },
|
||||||
{ name = "pydantic" },
|
{ name = "pydantic" },
|
||||||
{ name = "pyexecjs" },
|
{ name = "pyexecjs" },
|
||||||
|
{ name = "pymysql" },
|
||||||
{ name = "python-jose", extra = ["cryptography"] },
|
{ name = "python-jose", extra = ["cryptography"] },
|
||||||
{ name = "python-multipart" },
|
{ name = "python-multipart" },
|
||||||
{ name = "requests", extra = ["socks"] },
|
{ name = "requests", extra = ["socks"] },
|
||||||
@@ -238,6 +239,7 @@ requires-dist = [
|
|||||||
{ name = "pycryptodome", specifier = ">=3.19.0" },
|
{ name = "pycryptodome", specifier = ">=3.19.0" },
|
||||||
{ name = "pydantic", specifier = ">=2.0.0" },
|
{ name = "pydantic", specifier = ">=2.0.0" },
|
||||||
{ name = "pyexecjs", specifier = ">=1.5.1" },
|
{ name = "pyexecjs", specifier = ">=1.5.1" },
|
||||||
|
{ name = "pymysql", specifier = ">=1.1,<2" },
|
||||||
{ name = "python-jose", extras = ["cryptography"], specifier = ">=3.3.0" },
|
{ name = "python-jose", extras = ["cryptography"], specifier = ">=3.3.0" },
|
||||||
{ name = "python-multipart", specifier = ">=0.0.9" },
|
{ name = "python-multipart", specifier = ">=0.0.9" },
|
||||||
{ name = "requests", extras = ["socks"], specifier = ">=2.31.0" },
|
{ name = "requests", extras = ["socks"], specifier = ">=2.31.0" },
|
||||||
@@ -565,6 +567,15 @@ dependencies = [
|
|||||||
]
|
]
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/ba/8e/aedef81641c8dca6fd0fb7294de5bed9c45f3397d67fddf755c1042c2642/PyExecJS-1.5.1.tar.gz", hash = "sha256:34cc1d070976918183ff7bdc0ad71f8157a891c92708c00c5fbbff7a769f505c", size = 13344, upload-time = "2018-01-18T04:33:55.126Z" }
|
sdist = { url = "https://files.pythonhosted.org/packages/ba/8e/aedef81641c8dca6fd0fb7294de5bed9c45f3397d67fddf755c1042c2642/PyExecJS-1.5.1.tar.gz", hash = "sha256:34cc1d070976918183ff7bdc0ad71f8157a891c92708c00c5fbbff7a769f505c", size = 13344, upload-time = "2018-01-18T04:33:55.126Z" }
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pymysql"
|
||||||
|
version = "1.2.0"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/c9/bc/1c6a92f385940f727daeecf3bacaf186e03875dff57197801046c583bcf0/pymysql-1.2.0.tar.gz", hash = "sha256:6c7b17ca686988104d7426c27895b455cdeea3e9d3ceb1270f0c3704fead8c33", size = 49021, upload-time = "2026-05-19T08:26:22.302Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/c4/bd/2534e130295c8cfd4f0a2e31623baab7502278f1e97bcfe61db75656a77f/pymysql-1.2.0-py3-none-any.whl", hash = "sha256:62169ce6d5510f08e140c5e7990ee884a9764024e4a9a27b2cc11f1099322ae0", size = 45716, upload-time = "2026-05-19T08:26:20.974Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pysocks"
|
name = "pysocks"
|
||||||
version = "1.7.1"
|
version = "1.7.1"
|
||||||
|
|||||||
+50
-6
@@ -2,13 +2,48 @@
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from urllib.parse import quote_plus
|
||||||
|
|
||||||
from sqlalchemy import create_engine, event
|
from sqlalchemy import create_engine, event
|
||||||
from sqlalchemy.orm import sessionmaker, declarative_base
|
from sqlalchemy.orm import sessionmaker, declarative_base
|
||||||
|
|
||||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||||
DB_PATH = PROJECT_ROOT / "data" / "web.db"
|
DB_PATH = PROJECT_ROOT / "data" / "web.db"
|
||||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
DATABASE_URL = os.getenv("DATABASE_URL", f"sqlite:///{DB_PATH}")
|
|
||||||
|
|
||||||
|
def _env_int(name: str, default: int) -> int:
|
||||||
|
"""读取正整数环境变量,非法值回退到默认值。"""
|
||||||
|
try:
|
||||||
|
value = int(os.getenv(name, str(default)))
|
||||||
|
except ValueError:
|
||||||
|
return default
|
||||||
|
return value if value > 0 else default
|
||||||
|
|
||||||
|
|
||||||
|
def _get_database_url() -> str:
|
||||||
|
"""优先使用完整连接串,否则按 DB_* 环境变量构造 MySQL 连接。"""
|
||||||
|
database_url = os.getenv("DATABASE_URL", "").strip()
|
||||||
|
if database_url:
|
||||||
|
return database_url
|
||||||
|
|
||||||
|
db_host = os.getenv("DB_HOST", "").strip()
|
||||||
|
if db_host:
|
||||||
|
db_port = _env_int("DB_PORT", 3306)
|
||||||
|
db_name = os.getenv("DB_NAME", "douyu_login").strip()
|
||||||
|
db_user = os.getenv("DB_USER", "douyu_login").strip()
|
||||||
|
db_password = os.getenv("DB_PASSWORD", "")
|
||||||
|
if not db_name or not db_user or not db_password:
|
||||||
|
raise RuntimeError("使用 DB_HOST 时必须同时设置 DB_NAME、DB_USER 和 DB_PASSWORD")
|
||||||
|
return (
|
||||||
|
f"mysql+pymysql://{quote_plus(db_user)}:{quote_plus(db_password)}"
|
||||||
|
f"@{db_host}:{db_port}/{db_name}?charset=utf8mb4"
|
||||||
|
)
|
||||||
|
|
||||||
|
return f"sqlite:///{DB_PATH}"
|
||||||
|
|
||||||
|
|
||||||
|
DATABASE_URL = _get_database_url()
|
||||||
|
|
||||||
connect_args = (
|
connect_args = (
|
||||||
{"check_same_thread": False, "timeout": 30}
|
{"check_same_thread": False, "timeout": 30}
|
||||||
@@ -16,11 +51,20 @@ connect_args = (
|
|||||||
else {}
|
else {}
|
||||||
)
|
)
|
||||||
|
|
||||||
engine = create_engine(
|
engine_options = {
|
||||||
DATABASE_URL,
|
"connect_args": connect_args,
|
||||||
connect_args=connect_args,
|
"echo": False,
|
||||||
echo=False,
|
}
|
||||||
)
|
if not DATABASE_URL.startswith("sqlite"):
|
||||||
|
engine_options.update(
|
||||||
|
pool_pre_ping=True,
|
||||||
|
pool_size=_env_int("DB_POOL_SIZE", 20),
|
||||||
|
max_overflow=_env_int("DB_MAX_OVERFLOW", 20),
|
||||||
|
pool_timeout=_env_int("DB_POOL_TIMEOUT", 30),
|
||||||
|
pool_recycle=_env_int("DB_POOL_RECYCLE", 1800),
|
||||||
|
)
|
||||||
|
|
||||||
|
engine = create_engine(DATABASE_URL, **engine_options)
|
||||||
|
|
||||||
|
|
||||||
if DATABASE_URL.startswith("sqlite"):
|
if DATABASE_URL.startswith("sqlite"):
|
||||||
|
|||||||
@@ -0,0 +1,67 @@
|
|||||||
|
"""补充高频查询组合索引
|
||||||
|
|
||||||
|
Revision ID: 20260725_0009
|
||||||
|
Revises: 20260724_0008
|
||||||
|
Create Date: 2026-07-25
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision: str = "20260725_0009"
|
||||||
|
down_revision: Union[str, None] = "20260724_0008"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def _has_table(bind, table_name: str) -> bool:
|
||||||
|
return sa.inspect(bind).has_table(table_name)
|
||||||
|
|
||||||
|
|
||||||
|
def _indexes(bind, table_name: str) -> set[str]:
|
||||||
|
if not _has_table(bind, table_name):
|
||||||
|
return set()
|
||||||
|
return {index["name"] for index in sa.inspect(bind).get_indexes(table_name)}
|
||||||
|
|
||||||
|
|
||||||
|
def _create_index_if_missing(bind, name: str, table_name: str, columns: list[str]) -> None:
|
||||||
|
if name not in _indexes(bind, table_name):
|
||||||
|
op.create_index(name, table_name, columns)
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_index_if_exists(bind, name: str, table_name: str) -> None:
|
||||||
|
if name in _indexes(bind, table_name):
|
||||||
|
op.drop_index(name, table_name=table_name)
|
||||||
|
|
||||||
|
|
||||||
|
INDEXES = [
|
||||||
|
("ix_accounts_assigned_to_id", "accounts", ["assigned_to", "id"]),
|
||||||
|
("ix_accounts_tag_id", "accounts", ["tag", "id"]),
|
||||||
|
("ix_login_tasks_account_status_id", "login_tasks", ["account_id", "status", "id"]),
|
||||||
|
("ix_login_tasks_status_finished_at", "login_tasks", ["status", "finished_at"]),
|
||||||
|
("ix_douyu_tasks_account_id", "douyu_tasks", ["account_id", "id"]),
|
||||||
|
("ix_douyu_tasks_created_by_id", "douyu_tasks", ["created_by", "id"]),
|
||||||
|
("ix_douyu_tasks_batch_status_id", "douyu_tasks", ["batch_id", "status", "id"]),
|
||||||
|
("ix_huya_accounts_assigned_to_id", "huya_accounts", ["assigned_to", "id"]),
|
||||||
|
("ix_huya_accounts_updated_at_id", "huya_accounts", ["updated_at", "id"]),
|
||||||
|
("ix_huya_tasks_account_id", "huya_tasks", ["account_id", "id"]),
|
||||||
|
("ix_huya_tasks_created_by_id", "huya_tasks", ["created_by", "id"]),
|
||||||
|
("ix_huya_tasks_batch_status_id", "huya_tasks", ["batch_id", "status", "id"]),
|
||||||
|
("ix_huya_register_items_batch_db_line", "huya_register_items", ["batch_db_id", "line"]),
|
||||||
|
("ix_huya_register_success_logs_batch_id_id", "huya_register_success_logs", ["batch_id", "id"]),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
bind = op.get_bind()
|
||||||
|
for name, table_name, columns in INDEXES:
|
||||||
|
_create_index_if_missing(bind, name, table_name, columns)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
bind = op.get_bind()
|
||||||
|
for name, table_name, _ in reversed(INDEXES):
|
||||||
|
_drop_index_if_exists(bind, name, table_name)
|
||||||
Reference in New Issue
Block a user