377 lines
13 KiB
Bash
Executable File
377 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
# 数据库恢复与备份验证。
|
|
#
|
|
# 用法:
|
|
# bash deploy/restore-db.sh --list # 列出 OSS 中的数据库备份(倒序)
|
|
# bash deploy/restore-db.sh --verify # 用最新备份做恢复演练(临时容器,不影响线上库)
|
|
# bash deploy/restore-db.sh --verify --key backups/postgres/order_site-xxx.dump
|
|
# bash deploy/restore-db.sh # 交互确认后,恢复最新备份到线上库
|
|
# bash deploy/restore-db.sh --key backups/postgres/order_site-xxx.dump --yes
|
|
# bash deploy/restore-db.sh --restore-configs # 同时恢复最新 data 配置备份
|
|
# bash deploy/restore-db.sh --allow-unverified # 仅兼容历史无 manifest 备份
|
|
#
|
|
# 恢复流程:下载备份(校验 sha256)→ 停 backend/web → pg_restore --clean → 执行迁移 → 重启并等待健康检查。
|
|
# 详见 docs/数据库备份与恢复.md。
|
|
|
|
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
ENV_FILE="${PROJECT_ROOT}/.env"
|
|
TMP_DIR="${PROJECT_ROOT}/backups/tmp"
|
|
MODE="restore"
|
|
KEY=""
|
|
CONFIG_KEY=""
|
|
RESTORE_CONFIGS=false
|
|
ASSUME_YES="${RESTORE_CONFIRM:-}"
|
|
ALLOW_UNVERIFIED=false
|
|
BACKUP_PREFIX=""
|
|
CONFIG_PREFIX=""
|
|
RESTORE_ACTIVE=false
|
|
RESTORE_SUCCEEDED=false
|
|
BACKEND_WAS_RUNNING=false
|
|
WEB_WAS_RUNNING=false
|
|
RESTORE_TARGET=""
|
|
CONFIG_TARGET=""
|
|
VERIFY_CONTAINER=""
|
|
|
|
log() { printf '\033[1;34m[restore]\033[0m %s\n' "$*"; }
|
|
warn() { printf '\033[1;33m[restore]\033[0m %s\n' "$*"; }
|
|
fail() { printf '\033[1;31m[restore]\033[0m %s\n' "$*" >&2; exit 1; }
|
|
|
|
parse_args() {
|
|
while (($# > 0)); do
|
|
case "$1" in
|
|
--list|--verify)
|
|
[[ "${MODE}" == "restore" ]] || fail "--list 与 --verify 互斥。"
|
|
MODE="${1#--}"
|
|
shift
|
|
;;
|
|
--key)
|
|
[[ $# -ge 2 ]] || fail "--key 需要一个备份 key 参数。"
|
|
KEY="$2"
|
|
shift 2
|
|
;;
|
|
--config-key)
|
|
[[ $# -ge 2 ]] || fail "--config-key 需要一个配置备份 key 参数。"
|
|
CONFIG_KEY="$2"
|
|
RESTORE_CONFIGS=true
|
|
shift 2
|
|
;;
|
|
--restore-configs)
|
|
RESTORE_CONFIGS=true
|
|
shift
|
|
;;
|
|
--allow-unverified)
|
|
ALLOW_UNVERIFIED=true
|
|
shift
|
|
;;
|
|
--yes)
|
|
ASSUME_YES=1
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
sed -n '4,13p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
|
exit 0
|
|
;;
|
|
*)
|
|
fail "未知参数:$1。执行 bash deploy/restore-db.sh --help 查看用法。"
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
read_env_value() {
|
|
local key="$1"
|
|
[[ -f "${ENV_FILE}" ]] || return 0
|
|
awk -F= -v key="${key}" '
|
|
$0 !~ /^[[:space:]]*#/ && $1 == key {
|
|
value = substr($0, index($0, "=") + 1)
|
|
gsub(/^[[:space:]]+|[[:space:]]+$/, "", value)
|
|
gsub(/^"|"$/, "", value)
|
|
gsub(/^'\''|'\''$/, "", value)
|
|
print value
|
|
exit
|
|
}
|
|
' "${ENV_FILE}"
|
|
}
|
|
|
|
run_backup_tool() {
|
|
docker compose run --rm --no-deps -T backend node dist/db-backup.js "$@"
|
|
}
|
|
|
|
restore_cleanup() {
|
|
local exit_code=$?
|
|
if [[ "${RESTORE_ACTIVE}" == "true" && "${RESTORE_SUCCEEDED}" != "true" && "${exit_code}" -ne 0 ]]; then
|
|
warn "恢复失败,尝试恢复恢复前正在运行的业务服务。数据库可能处于部分恢复状态,请立即检查。"
|
|
if [[ "${BACKEND_WAS_RUNNING}" == "true" ]]; then
|
|
docker compose up -d --no-deps backend >/dev/null 2>&1 || warn "backend 自动重启失败。"
|
|
fi
|
|
if [[ "${WEB_WAS_RUNNING}" == "true" ]]; then
|
|
docker compose up -d --no-deps web >/dev/null 2>&1 || warn "web 自动重启失败。"
|
|
fi
|
|
fi
|
|
[[ -n "${RESTORE_TARGET}" ]] && rm -f -- "${RESTORE_TARGET}"
|
|
[[ -n "${CONFIG_TARGET}" ]] && rm -f -- "${CONFIG_TARGET}"
|
|
[[ -n "${VERIFY_CONTAINER}" ]] && docker rm -f -v "${VERIFY_CONTAINER}" >/dev/null 2>&1 || true
|
|
trap - EXIT
|
|
exit "${exit_code}"
|
|
}
|
|
|
|
is_external_database() {
|
|
local database_url
|
|
database_url="$(read_env_value DATABASE_URL)"
|
|
[[ -n "${database_url}" && "${database_url}" != *"@postgres:"* ]]
|
|
}
|
|
|
|
resolve_key() {
|
|
if [[ -n "${KEY}" ]]; then
|
|
printf '%s\n' "${KEY}"
|
|
return
|
|
fi
|
|
local latest
|
|
local latest_args=(latest --prefix "${BACKUP_PREFIX}")
|
|
[[ "${ALLOW_UNVERIFIED}" == "true" ]] && latest_args+=(--allow-unverified)
|
|
if ! latest="$(run_backup_tool "${latest_args[@]}")"; then
|
|
fail "未能获取最新备份 key。可先用 bash deploy/restore-db.sh --list 查看已有备份。"
|
|
fi
|
|
printf '%s\n' "${latest}"
|
|
}
|
|
|
|
resolve_config_key() {
|
|
if [[ -n "${CONFIG_KEY}" ]]; then
|
|
printf '%s\n' "${CONFIG_KEY}"
|
|
return
|
|
fi
|
|
local latest
|
|
latest="$(run_backup_tool list --prefix "${CONFIG_PREFIX}" | awk -F '\t' '$1 ~ /\.tar\.gz$/ { print $1; exit }' || true)"
|
|
[[ -n "${latest}" ]] || fail "未找到配置备份。可使用 --config-key 指定配置备份,或不启用 --restore-configs。"
|
|
printf '%s\n' "${latest}"
|
|
}
|
|
|
|
download_backup() {
|
|
local key="$1" target="$2" kind="${3:-postgres-dump}"
|
|
log "下载备份并校验 sha256:${key}"
|
|
local verification_args=(--key "${key}" --kind "${kind}")
|
|
[[ "${ALLOW_UNVERIFIED}" == "true" ]] && verification_args+=(--allow-unverified)
|
|
if ! run_backup_tool download "${verification_args[@]}" > "${target}"; then
|
|
rm -f "${target}"
|
|
fail "下载或校验失败:${key}"
|
|
fi
|
|
[[ -s "${target}" ]] || fail "下载内容为空:${key}"
|
|
}
|
|
|
|
run_pg_restore_stdin() {
|
|
local database_url
|
|
database_url="$(read_env_value DATABASE_URL)"
|
|
if [[ -z "${database_url}" || "${database_url}" == *"@postgres:"* ]]; then
|
|
docker compose exec -T postgres sh -c 'pg_restore -U "${POSTGRES_USER:-postgres}" -d "${POSTGRES_DB:-order_site}" --clean --if-exists --no-owner'
|
|
else
|
|
log "DATABASE_URL 指向外部数据库,从 postgres 容器经网络直连恢复"
|
|
docker compose exec -T -e DB_RESTORE_URL="${database_url}" postgres sh -c 'pg_restore --dbname="$DB_RESTORE_URL" --clean --if-exists --no-owner'
|
|
fi
|
|
}
|
|
|
|
service_is_running() {
|
|
local service="$1" container_id status
|
|
container_id="$(docker compose ps -q "${service}" 2>/dev/null || true)"
|
|
[[ -n "${container_id}" ]] || return 1
|
|
status="$(docker inspect -f '{{.State.Status}}' "${container_id}" 2>/dev/null || true)"
|
|
[[ "${status}" == "running" ]]
|
|
}
|
|
|
|
restore_configs() {
|
|
local source="$1" staging="${TMP_DIR}/config-extract-$$"
|
|
log "恢复配置备份到 apps/backend/data"
|
|
while IFS= read -r entry; do
|
|
case "${entry}" in
|
|
/*|../*|*/../*|..)
|
|
fail "配置备份包含不安全路径:${entry}"
|
|
;;
|
|
esac
|
|
done < <(tar -tzf "${source}")
|
|
rm -rf -- "${staging}"
|
|
mkdir -p "${staging}"
|
|
if ! tar -xzf "${source}" --no-same-owner -C "${staging}"; then
|
|
rm -rf -- "${staging}"
|
|
fail "配置备份解包失败,未修改现有配置。"
|
|
fi
|
|
cp -a "${staging}/." "${PROJECT_ROOT}/apps/backend/data/"
|
|
rm -rf -- "${staging}"
|
|
}
|
|
|
|
restore_live_database() {
|
|
local key="$1" target="$2" config_target="${3:-}"
|
|
|
|
warn "即将用备份覆盖当前数据库:${key}"
|
|
warn "当前库中的订单、后台用户、审计日志等数据都会被备份内容替换。"
|
|
if [[ "${ASSUME_YES}" != "1" ]]; then
|
|
if [[ ! -t 0 ]]; then
|
|
fail "当前不是交互式终端。确认恢复请执行:RESTORE_CONFIRM=1 bash deploy/restore-db.sh --key ${key}"
|
|
fi
|
|
local answer
|
|
printf '请输入 RESTORE 确认恢复数据库:'
|
|
read -r answer
|
|
[[ "${answer}" == "RESTORE" ]] || fail "未确认恢复,已停止。"
|
|
fi
|
|
|
|
BACKEND_WAS_RUNNING=false
|
|
WEB_WAS_RUNNING=false
|
|
service_is_running backend && BACKEND_WAS_RUNNING=true
|
|
service_is_running web && WEB_WAS_RUNNING=true
|
|
RESTORE_ACTIVE=true
|
|
log "停止 backend/web,避免恢复期间产生写入"
|
|
docker compose stop backend web >/dev/null 2>&1 || true
|
|
if service_is_running backend || service_is_running web; then
|
|
fail "无法确认 backend/web 已停止,为避免恢复期间写入已中止。"
|
|
fi
|
|
|
|
log "pg_restore 恢复中(--clean --if-exists)"
|
|
if ! run_pg_restore_stdin < "${target}"; then
|
|
fail "pg_restore 失败。数据库处于中间状态,请根据日志处理;必要时用 --list 选择更早的备份重试。"
|
|
fi
|
|
|
|
log "执行数据库迁移(补齐备份之后新增的迁移)"
|
|
if ! docker compose run --rm --no-deps backend node dist/db/migrate.js; then
|
|
fail "迁移失败。请查看日志;数据已恢复到备份时点。"
|
|
fi
|
|
|
|
if [[ -n "${config_target}" ]]; then
|
|
restore_configs "${config_target}"
|
|
fi
|
|
|
|
log "重启 backend/web"
|
|
docker compose up -d --no-deps backend >/dev/null || fail "backend 启动失败。"
|
|
wait_backend_healthy
|
|
docker compose up -d --no-deps web >/dev/null || fail "web 启动失败。"
|
|
|
|
local health_url site_addr
|
|
site_addr="$(read_env_value CADDY_SITE_ADDR)"
|
|
health_url="${site_addr%/}/health/ready"
|
|
if curl -fsS "${health_url}" >/dev/null 2>&1; then
|
|
log "健康检查通过:${health_url}"
|
|
else
|
|
fail "健康检查未通过:${health_url},请检查 docker compose logs backend。"
|
|
fi
|
|
RESTORE_SUCCEEDED=true
|
|
log "恢复完成。建议抽查近期订单与后台登录确认数据正确。"
|
|
}
|
|
|
|
wait_backend_healthy() {
|
|
local attempt container_id status
|
|
for ((attempt = 1; attempt <= 60; attempt++)); do
|
|
container_id="$(docker compose ps -q backend 2>/dev/null || true)"
|
|
if [[ -n "${container_id}" ]]; then
|
|
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{end}}' "${container_id}" 2>/dev/null || true)"
|
|
if [[ "${status}" == "healthy" ]]; then
|
|
return
|
|
fi
|
|
fi
|
|
sleep 2
|
|
done
|
|
fail "等待 backend 健康超时(120 秒),请检查 docker compose logs backend。"
|
|
}
|
|
|
|
resolve_postgres_image() {
|
|
local container_id image
|
|
container_id="$(docker compose ps -q postgres 2>/dev/null || true)"
|
|
if [[ -n "${container_id}" ]]; then
|
|
image="$(docker inspect -f '{{.Config.Image}}' "${container_id}" 2>/dev/null || true)"
|
|
if [[ -n "${image}" ]]; then
|
|
printf '%s\n' "${image}"
|
|
return
|
|
fi
|
|
fi
|
|
image="$(docker compose images postgres -q 2>/dev/null | head -n 1 || true)"
|
|
[[ -n "${image}" ]] || fail "无法确定 postgres 镜像(线上 postgres 未运行)。"
|
|
printf '%s\n' "${image}"
|
|
}
|
|
|
|
verify_backup() {
|
|
local key="$1" target="$2" image cid attempt
|
|
|
|
image="$(resolve_postgres_image)"
|
|
log "使用镜像 ${image} 启动临时验证容器(不接入任何业务网络)"
|
|
|
|
docker rm -f order-site-backup-verify >/dev/null 2>&1 || true
|
|
cid="$(docker run -d --name order-site-backup-verify -e POSTGRES_PASSWORD=verify-only "${image}")"
|
|
VERIFY_CONTAINER="order-site-backup-verify"
|
|
|
|
for ((attempt = 1; attempt <= 30; attempt++)); do
|
|
if docker exec "${cid}" pg_isready -U postgres >/dev/null 2>&1; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
docker exec "${cid}" pg_isready -U postgres >/dev/null 2>&1 || fail "临时验证容器启动超时。"
|
|
|
|
log "在临时容器中恢复备份"
|
|
docker cp "${target}" "${cid}:/tmp/verify.dump" >/dev/null
|
|
docker exec "${cid}" createdb -U postgres order_site >/dev/null
|
|
if ! docker exec "${cid}" pg_restore -U postgres -d order_site --no-owner /tmp/verify.dump; then
|
|
fail "备份无法恢复:pg_restore 报错,该备份不可用。"
|
|
fi
|
|
|
|
log "备份可恢复。已应用迁移:"
|
|
docker exec "${cid}" psql -U postgres -d order_site -At -c 'SELECT filename FROM schema_migrations ORDER BY filename'
|
|
|
|
log "各表行数:"
|
|
docker exec "${cid}" psql -U postgres -d order_site -At -F '=' \
|
|
-c "SELECT tablename, (xpath('/row/count/text()', query_to_xml(format('SELECT count(*) AS count FROM %I.%I', schemaname, tablename), true, false, '')))[1]::text::bigint FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename"
|
|
|
|
log "验证通过,已清理临时容器。建议每月执行一次。"
|
|
docker rm -f -v "${VERIFY_CONTAINER}" >/dev/null 2>&1 || true
|
|
VERIFY_CONTAINER=""
|
|
}
|
|
|
|
list_backups() {
|
|
log "OSS 中的数据库备份(key/字节/时间):"
|
|
run_backup_tool list --prefix "${BACKUP_PREFIX}"
|
|
}
|
|
|
|
main() {
|
|
parse_args "$@"
|
|
cd "${PROJECT_ROOT}"
|
|
|
|
command -v docker >/dev/null 2>&1 || fail "未找到 docker。"
|
|
docker compose version >/dev/null 2>&1 || fail "未找到 docker compose 插件。"
|
|
|
|
BACKUP_PREFIX="${DB_BACKUP_OSS_PREFIX:-$(read_env_value DB_BACKUP_OSS_PREFIX || true)}"
|
|
BACKUP_PREFIX="${BACKUP_PREFIX%/}"
|
|
BACKUP_PREFIX="${BACKUP_PREFIX:-backups/postgres}"
|
|
[[ "${BACKUP_PREFIX}" != /* && "${BACKUP_PREFIX}" != *'..'* ]] || fail "DB_BACKUP_OSS_PREFIX 不能是绝对路径或包含 ..。"
|
|
CONFIG_PREFIX="${BACKUP_PREFIX%/postgres}/configs"
|
|
|
|
if [[ "${MODE}" == "list" ]]; then
|
|
list_backups
|
|
return
|
|
fi
|
|
|
|
mkdir -p "${TMP_DIR}"
|
|
local key target config_key
|
|
key="$(resolve_key)"
|
|
target="${TMP_DIR}/$(basename "${key}")"
|
|
RESTORE_TARGET="${target}"
|
|
|
|
download_backup "${key}" "${target}" postgres-dump
|
|
|
|
if [[ "${RESTORE_CONFIGS}" == "true" && "${MODE}" == "restore" ]]; then
|
|
config_key="$(resolve_config_key)"
|
|
CONFIG_TARGET="${TMP_DIR}/$(basename "${config_key}")"
|
|
download_backup "${config_key}" "${CONFIG_TARGET}" app-configs
|
|
fi
|
|
|
|
if [[ "${MODE}" == "verify" ]]; then
|
|
verify_backup "${key}" "${target}"
|
|
rm -f -- "${target}"
|
|
return
|
|
fi
|
|
|
|
if is_external_database; then
|
|
warn "DATABASE_URL 指向外部数据库:pg_restore 将经网络直连外部库并覆盖其数据,请确认目标正确。"
|
|
fi
|
|
restore_live_database "${key}" "${target}" "${CONFIG_TARGET:-}"
|
|
}
|
|
|
|
trap restore_cleanup EXIT
|
|
main "$@"
|