yml/hfb_sys
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

重写在线备份链路

Browse Source
This commit is contained in:
yml2213
2026-08-16 22:32:07 +08:00
parent f48da14ed2
commit 7438fb6725
10 changed files with 999 additions and 608 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/archive-binlog.sh
Executable
+175
View File
@@ -0,0 +1,175 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
# 分钟级 binlog 归档。每次轮转后只归档已关闭文件；每个对象以
# server_uuid、原始文件名和原始 sha256 命名，避免实例重建后的编号碰撞。
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=backup-common.sh
source "${SCRIPT_DIR}/backup-common.sh"
MYSQL_DATA_DIR='/var/lib/mysql'
work_dir=''
cleanup() {
if [[ -n "${work_dir}" && -d "${work_dir}" ]]; then
rm -rf -- "${work_dir}"
fi
}
trap cleanup EXIT
require_archive_config() {
backup_require_env BACKUP_DIR >/dev/null
backup_require_env BACKUP_OSS_URI >/dev/null
backup_require_env BACKUP_PASSPHRASE >/dev/null
backup_require_env BACKUP_KMS_KEY_ID >/dev/null
}
closed_binlogs() {
local mysql_cid="$1"
docker exec "${mysql_cid}" sh -c \
'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket -N -B -e "SHOW BINARY LOGS;"' \
2>/dev/null | awk 'NR > 1 { print previous } { previous = $1 } END { }'
}
state_last_index() {
local state_dir="$1"
local value
[[ -f "${state_dir}/last-archived.env" ]] || return
value="$(awk -F= '$1 == "binlog_index" { print $2; exit }' "${state_dir}/last-archived.env")"
[[ "${value}" =~ ^[0-9]+$ ]] && printf '%s' "${value}"
}
write_state() {
local state_dir="$1" filename="$2" index="$3" raw_sha="$4" cipher_sha="$5" remote_dir="$6"
{
printf 'filename=%s\n' "${filename}"
printf 'binlog_index=%s\n' "${index}"
printf 'raw_sha256=%s\n' "${raw_sha}"
printf 'cipher_sha256=%s\n' "${cipher_sha}"
printf 'remote_dir=%s\n' "${remote_dir}"
} > "${state_dir}/${filename}.${raw_sha}.state"
cp "${state_dir}/${filename}.${raw_sha}.state" "${state_dir}/last-archived.env"
}
archive_one() {
local mysql_cid="$1" state_dir="$2" server_uuid="$3" oss_uri="$4" filename="$5"
local source_path source_sha raw_file raw_sha cipher_file cipher_sha cipher_size binlog_index object_id remote_dir
source_path="${MYSQL_DATA_DIR}/${filename}"
if ! docker exec "${mysql_cid}" test -f "${source_path}" >/dev/null 2>&1; then
backup_error "待归档 binlog 已从 MySQL 数据目录消失：${filename}；PITR 链路已断裂"
exit 1
fi
raw_file="${work_dir}/${filename}"
docker cp "${mysql_cid}:${source_path}" "${raw_file}"
raw_sha="$(backup_sha256 "${raw_file}")"
source_sha="$(docker exec "${mysql_cid}" sha256sum "${source_path}" | awk '{print $1}')"
[[ "${source_sha}" == "${raw_sha}" ]] || {
backup_error "binlog 拷贝校验失败：${filename}"
exit 1
}
if compgen -G "${state_dir}/${filename}.${raw_sha}.state" >/dev/null; then
return
fi
binlog_index="$(backup_binlog_index "${filename}")"
object_id="${filename}.${raw_sha}"
remote_dir="${oss_uri}/mysql/binlog/${server_uuid}/${object_id}"
cipher_file="${work_dir}/${object_id}.enc"
backup_encrypt_file "${raw_file}" "${cipher_file}"
cipher_sha="$(backup_sha256 "${cipher_file}")"
cipher_size="$(backup_file_size "${cipher_file}")"
printf '%s %s\n' "${cipher_sha}" 'binlog.enc' > "${work_dir}/${object_id}.enc.sha256"
{
printf 'format_version=1\n'
printf 'server_uuid=%s\n' "${server_uuid}"
printf 'source_binlog=%s\n' "${filename}"
printf 'binlog_index=%s\n' "${binlog_index}"
printf 'raw_sha256=%s\n' "${raw_sha}"
printf 'payload=binlog.enc\n'
printf 'payload_sha256=%s\n' "${cipher_sha}"
printf 'payload_size_bytes=%s\n' "${cipher_size}"
printf 'created_at=%s\n' "$(date -Iseconds)"
} > "${work_dir}/${object_id}.complete.env"
backup_oss_upload_kms "${cipher_file}" "${remote_dir}/binlog.enc"
backup_oss_upload_kms "${work_dir}/${object_id}.enc.sha256" "${remote_dir}/binlog.enc.sha256"
# complete.env 最后上传，恢复程序只使用有该文件的 binlog。
backup_oss_upload_kms "${work_dir}/${object_id}.complete.env" "${remote_dir}/complete.env"
write_state "${state_dir}" "${filename}" "${binlog_index}" "${raw_sha}" "${cipher_sha}" "${remote_dir}"
rm -f -- "${raw_file}" "${cipher_file}" "${work_dir}/${object_id}.enc.sha256" "${work_dir}/${object_id}.complete.env"
}
archive() {
backup_require_cmd docker
backup_require_cmd ossutil
backup_require_cmd openssl
require_archive_config
local backup_dir oss_uri mysql_cid server_uuid state_dir first_binlog last_index first_index expected_index filename index
backup_dir="$(backup_require_env BACKUP_DIR)"
backup_validate_dir "${backup_dir}"
oss_uri="$(backup_require_env BACKUP_OSS_URI)"
backup_validate_oss_uri "${oss_uri}"
backup_lock_or_exit "${backup_dir}/.binlog.lock"
mysql_cid="$(backup_mysql_container_id)"
server_uuid="$(backup_server_uuid)"
state_dir="${backup_dir}/binlog-state/${server_uuid}"
work_dir="${backup_dir}/.tmp-binlog-$$"
mkdir -p "${state_dir}" "${work_dir}"
docker exec "${mysql_cid}" sh -c \
'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket -e "FLUSH BINARY LOGS;"' >/dev/null
first_binlog="$(docker exec "${mysql_cid}" sh -c \
'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket -N -B -e "SHOW BINARY LOGS;"' | head -1 | awk '{print $1}')"
[[ -n "${first_binlog}" ]] || { backup_error '无法读取 binlog 列表'; exit 1; }
last_index="$(state_last_index "${state_dir}" || true)"
first_index="$(backup_binlog_index "${first_binlog}")"
if [[ -n "${last_index}" && "${first_index}" -gt $((last_index + 1)) ]]; then
backup_error "发现未归档 binlog 缺口：已归档至序号 ${last_index}，当前最早为 ${first_index}"
exit 1
fi
expected_index="${last_index}"
while IFS= read -r filename; do
[[ -n "${filename}" ]] || continue
index="$(backup_binlog_index "${filename}")"
if [[ -n "${expected_index}" && "${index}" -le "${expected_index}" ]]; then
continue
fi
if [[ -n "${expected_index}" && "${index}" -ne $((expected_index + 1)) ]]; then
backup_error "发现未归档 binlog 缺口：期望序号 $((expected_index + 1))，实际为 ${index}"
exit 1
fi
archive_one "${mysql_cid}" "${state_dir}" "${server_uuid}" "${oss_uri}" "${filename}"
expected_index="${index}"
done < <(closed_binlogs "${mysql_cid}")
backup_log "binlog 归档完成，实例 ${server_uuid}"
}
verify() {
backup_require_cmd ossutil
require_archive_config
local backup_dir server_uuid state_dir state_file remote_dir
backup_dir="$(backup_require_env BACKUP_DIR)"
backup_validate_dir "${backup_dir}"
backup_lock_or_exit "${backup_dir}/.binlog.lock"
server_uuid="$(backup_server_uuid)"
state_dir="${backup_dir}/binlog-state/${server_uuid}"
[[ -d "${state_dir}" ]] || { backup_error '尚未产生 binlog 归档状态'; exit 1; }
while IFS= read -r -d '' state_file; do
remote_dir="$(awk -F= '$1 == "remote_dir" { print substr($0, index($0, "=") + 1); exit }' "${state_file}")"
[[ -n "${remote_dir}" ]] || { backup_error "归档状态不完整：${state_file}"; exit 1; }
ossutil stat "${remote_dir}/binlog.enc" >/dev/null
ossutil stat "${remote_dir}/binlog.enc.sha256" >/dev/null
ossutil stat "${remote_dir}/complete.env" >/dev/null
done < <(find "${state_dir}" -maxdepth 1 -name '*.state' -print0)
backup_log '全部本地 binlog 归档状态均有远端完整对象'
}
case "${1:-archive}" in
archive) archive ;;
verify) verify ;;
*)
printf '用法: %s {archive|verify}\n' "$0" >&2
exit 1
;;
esac
scripts/backup-common.sh
Executable
+193
View File
@@ -0,0 +1,193 @@
#!/usr/bin/env bash
# 在线备份脚本共用函数。仅供 scripts/backup-online.sh、archive-binlog.sh 和
# restore-online.sh source，不要单独执行。
BACKUP_ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
BACKUP_COMPOSE_FILE="${BACKUP_ROOT_DIR}/deploy/docker-compose.prod.yml"
BACKUP_ENV_FILE="${BACKUP_ROOT_DIR}/backend/.env"
BACKUP_XTRABACKUP_IMAGE="${XTRABACKUP_IMAGE:-percona/percona-xtrabackup:8.4}"
backup_log() {
printf '[backup] %s\n' "$*" >&2
}
backup_error() {
printf '[backup] ERROR: %s\n' "$*" >&2
}
backup_require_cmd() {
if ! command -v "$1" >/dev/null 2>&1; then
backup_error "缺少命令：$1"
exit 1
fi
}
backup_env_value() {
local key="$1"
awk -F= -v key="${key}" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' "${BACKUP_ENV_FILE}"
}
backup_require_env() {
local key="$1"
local value
value="$(backup_env_value "${key}")"
if [[ -z "${value}" ]]; then
backup_error "${BACKUP_ENV_FILE} 缺少或未填写 ${key}"
exit 1
fi
printf '%s' "${value}"
}
backup_compose() {
docker compose --env-file "${BACKUP_ENV_FILE}" -f "${BACKUP_COMPOSE_FILE}" "$@"
}
backup_mysql_container_id() {
local cid
cid="$(backup_compose ps -q mysql 2>/dev/null || true)"
if [[ -z "${cid}" ]]; then
backup_error "MySQL 容器未运行"
exit 1
fi
printf '%s' "${cid}"
}
backup_mysql_network() {
local network
network="$(docker inspect -f '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$1")"
if [[ -z "${network}" ]]; then
backup_error "无法读取 MySQL Docker 网络"
exit 1
fi
printf '%s' "${network}"
}
backup_mysql_root_query() {
backup_compose exec -T mysql sh -c \
'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket -N -B "$MYSQL_DATABASE" -e "$1"' \
sh "$1"
}
backup_validate_dir() {
local dir="$1"
if [[ "${dir}" != /data/* || "${dir}" == "/data" ]]; then
backup_error "备份目录必须是 /data 之下的具体绝对路径，当前值：${dir}"
exit 1
fi
}
backup_lock_or_exit() {
local path="$1"
local wait_seconds="${2:-${BACKUP_LOCK_WAIT_SECONDS:-0}}"
backup_require_cmd flock
mkdir -p "$(dirname "${path}")"
exec 9>"${path}"
if [[ ! "${wait_seconds}" =~ ^[0-9]+$ ]]; then
backup_error "锁等待时间必须是非负整数：${wait_seconds}"
exit 1
fi
if ! flock -w "${wait_seconds}" 9; then
backup_error "已有同类备份任务在执行，拒绝并发运行"
exit 1
fi
}
backup_server_uuid() {
local value
value="$(backup_mysql_root_query 'SELECT @@server_uuid;' 2>/dev/null || true)"
if [[ ! "${value}" =~ ^[0-9A-Fa-f-]{36}$ ]]; then
backup_error "无法读取 MySQL server_uuid"
exit 1
fi
printf '%s' "${value,,}"
}
backup_binlog_index() {
local filename="$1"
if [[ "${filename}" =~ ^mysql-bin\.([0-9]{6,})$ ]]; then
printf '%d' "$((10#${BASH_REMATCH[1]}))"
return
fi
backup_error "不支持的 binlog 文件名：${filename}"
exit 1
}
backup_sha256() {
if command -v sha256sum >/dev/null 2>&1; then
sha256sum "$1" | awk '{print $1}'
return
fi
shasum -a 256 "$1" | awk '{print $1}'
}
backup_file_size() {
if [[ "$(uname)" == 'Darwin' ]]; then
stat -f%z "$1"
else
stat -c%s "$1"
fi
}
backup_encrypt_file() {
local source="$1"
local destination="$2"
local passphrase
passphrase="$(backup_require_env BACKUP_PASSPHRASE)"
BACKUP_PASSPHRASE="${passphrase}" openssl enc -aes-256-cbc -pbkdf2 -iter 600000 -salt \
-pass env:BACKUP_PASSPHRASE \
-in "${source}" \
-out "${destination}"
unset passphrase
}
backup_decrypt_file() {
local source="$1"
local destination="$2"
local passphrase
passphrase="$(backup_require_env BACKUP_PASSPHRASE)"
BACKUP_PASSPHRASE="${passphrase}" openssl enc -d -aes-256-cbc -pbkdf2 -iter 600000 \
-pass env:BACKUP_PASSPHRASE \
-in "${source}" \
-out "${destination}"
unset passphrase
}
backup_oss_upload_kms() {
local source="$1"
local destination="$2"
local kms_key_id
kms_key_id="$(backup_require_env BACKUP_KMS_KEY_ID)"
ossutil cp "${source}" "${destination}" \
--encryption KMS --kms-master-key-id "${kms_key_id}"
}
backup_validate_oss_uri() {
local uri="$1"
if [[ ! "${uri}" =~ ^oss://[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
backup_error "BACKUP_OSS_URI 必须是 oss://bucket 格式，当前值：${uri}"
exit 1
fi
}
backup_oss_download() {
local source="$1"
local destination="$2"
ossutil cp "${source}" "${destination}"
}
backup_write_complete_marker() {
local path="$1"
local payload_name="$2"
local payload_sha256="$3"
local payload_size="$4"
local server_uuid="$5"
{
printf 'format_version=1\n'
printf 'created_at=%s\n' "$(date -Iseconds)"
printf 'server_uuid=%s\n' "${server_uuid}"
printf 'payload=%s\n' "${payload_name}"
printf 'payload_sha256=%s\n' "${payload_sha256}"
printf 'payload_size_bytes=%s\n' "${payload_size}"
} > "${path}"
}
scripts/backup-online.sh
Executable
+206
View File
@@ -0,0 +1,206 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
# MySQL 在线全量物理备份。XtraBackup 在业务读写期间运行，备份只在
# 上传 payload、manifest、checksum 后最后写入 complete.env，恢复程序只接受
# 带 complete.env 的备份集。
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=backup-common.sh
source "${SCRIPT_DIR}/backup-common.sh"
cleanup_dir=''
cleanup() {
if [[ -n "${cleanup_dir}" && -d "${cleanup_dir}" ]]; then
rm -rf -- "${cleanup_dir}"
fi
}
trap cleanup EXIT
require_backup_config() {
backup_require_env BACKUP_DIR >/dev/null
backup_require_env BACKUP_OSS_URI >/dev/null
backup_require_env BACKUP_PASSPHRASE >/dev/null
backup_require_env BACKUP_KMS_KEY_ID >/dev/null
backup_require_env BACKUP_MYSQL_USER >/dev/null
backup_require_env BACKUP_MYSQL_PASSWORD >/dev/null
}
upload_backup_set() {
local local_dir="$1"
local level="$2"
local ts="$3"
local server_uuid="$4"
local oss_uri="$5"
local remote_dir="${oss_uri}/mysql/full/${level}/${server_uuid}/${ts}"
backup_log "上传 ${level} 全量备份到 ${remote_dir}"
backup_oss_upload_kms "${local_dir}/mysql-full.tgz.enc" "${remote_dir}/mysql-full.tgz.enc"
backup_oss_upload_kms "${local_dir}/manifest.env" "${remote_dir}/manifest.env"
backup_oss_upload_kms "${local_dir}/mysql-full.tgz.enc.sha256" "${remote_dir}/mysql-full.tgz.enc.sha256"
# complete.env 必须最后上传，作为远端原子完成标识。
backup_oss_upload_kms "${local_dir}/complete.env" "${remote_dir}/complete.env"
}
full_backup() {
backup_require_cmd docker
backup_require_cmd openssl
backup_require_cmd ossutil
require_backup_config
local backup_dir oss_uri mysql_cid network server_uuid backup_user backup_password ts tmp_dir target_dir
backup_dir="$(backup_require_env BACKUP_DIR)"
backup_validate_dir "${backup_dir}"
oss_uri="$(backup_require_env BACKUP_OSS_URI)"
backup_validate_oss_uri "${oss_uri}"
backup_lock_or_exit "${backup_dir}/.full.lock"
mysql_cid="$(backup_mysql_container_id)"
network="$(backup_mysql_network "${mysql_cid}")"
server_uuid="$(backup_server_uuid)"
backup_user="$(backup_require_env BACKUP_MYSQL_USER)"
backup_password="$(backup_require_env BACKUP_MYSQL_PASSWORD)"
ts="$(date +%Y%m%d_%H%M%S)"
tmp_dir="${backup_dir}/.tmp-full-${ts}-$$"
target_dir="${backup_dir}/full/daily/${ts}"
cleanup_dir="${tmp_dir}"
mkdir -p "${tmp_dir}/xtrabackup"
cat > "${tmp_dir}/xtrabackup-client.cnf" <<EOF
[client]
user=${backup_user}
password=${backup_password}
host=mysql
EOF
chmod 600 "${tmp_dir}/xtrabackup-client.cnf"
backup_log "开始全量热备，实例 ${server_uuid}"
docker run --rm \
--network "${network}" \
--volumes-from "${mysql_cid}:ro" \
-v "${tmp_dir}/xtrabackup:/backup" \
-v "${tmp_dir}/xtrabackup-client.cnf:/etc/xtrabackup-client.cnf:ro" \
"${BACKUP_XTRABACKUP_IMAGE}" \
xtrabackup \
--defaults-extra-file=/etc/xtrabackup-client.cnf \
--backup \
--target-dir=/backup/data
rm -f "${tmp_dir}/xtrabackup-client.cnf"
unset backup_password
local binlog_file binlog_pos
binlog_file="$(awk -F$'\t' 'NR == 1 { print $1 }' "${tmp_dir}/xtrabackup/data/xtrabackup_binlog_info")"
binlog_pos="$(awk -F$'\t' 'NR == 1 { print $2 }' "${tmp_dir}/xtrabackup/data/xtrabackup_binlog_info")"
if [[ -z "${binlog_file}" || ! "${binlog_pos}" =~ ^[0-9]+$ ]]; then
backup_error "未读取到有效 xtrabackup_binlog_info"
exit 1
fi
{
printf 'format_version=1\n'
printf 'created_at=%s\n' "$(date -Iseconds)"
printf 'host=%s\n' "$(hostname)"
printf 'server_uuid=%s\n' "${server_uuid}"
printf 'mysql_version=%s\n' "$(backup_mysql_root_query 'SELECT VERSION();')"
printf 'goose_version=%s\n' "$(backup_mysql_root_query 'SELECT COALESCE(MAX(version_id), 0) FROM goose_db_version;')"
printf 'binlog_file=%s\n' "${binlog_file}"
printf 'binlog_pos=%s\n' "${binlog_pos}"
printf 'git_commit=%s\n' "$(git -C "${BACKUP_ROOT_DIR}" rev-parse HEAD 2>/dev/null || true)"
} > "${tmp_dir}/manifest.env"
# 关闭包含 XtraBackup 位点的当前 binlog 并归档。全量备份只有在该文件
# 已有远端 complete 标识后才会被标记完成。
BACKUP_LOCK_WAIT_SECONDS=120 "${SCRIPT_DIR}/archive-binlog.sh" archive
if ! find "${backup_dir}/binlog-state/${server_uuid}" -maxdepth 1 -type f \
-name "${binlog_file}.*.state" -print -quit | grep -q .; then
backup_error "XtraBackup 位点 ${binlog_file}:${binlog_pos} 尚未完成 binlog 归档"
exit 1
fi
tar czf "${tmp_dir}/mysql-full.tgz" \
-C "${tmp_dir}/xtrabackup/data" . \
-C "${tmp_dir}" manifest.env
backup_encrypt_file "${tmp_dir}/mysql-full.tgz" "${tmp_dir}/mysql-full.tgz.enc"
rm -f "${tmp_dir}/mysql-full.tgz"
local payload_sha payload_size
payload_sha="$(backup_sha256 "${tmp_dir}/mysql-full.tgz.enc")"
payload_size="$(backup_file_size "${tmp_dir}/mysql-full.tgz.enc")"
printf '%s %s\n' "${payload_sha}" 'mysql-full.tgz.enc' > "${tmp_dir}/mysql-full.tgz.enc.sha256"
backup_write_complete_marker "${tmp_dir}/complete.env" 'mysql-full.tgz.enc' "${payload_sha}" "${payload_size}" "${server_uuid}"
mkdir -p "${target_dir}"
mv "${tmp_dir}/mysql-full.tgz.enc" "${target_dir}/"
mv "${tmp_dir}/mysql-full.tgz.enc.sha256" "${target_dir}/"
mv "${tmp_dir}/manifest.env" "${target_dir}/"
mv "${tmp_dir}/complete.env" "${target_dir}/"
rm -rf "${tmp_dir}/xtrabackup"
cleanup_dir=''
rmdir "${tmp_dir}"
upload_backup_set "${target_dir}" daily "${ts}" "${server_uuid}" "${oss_uri}"
if [[ "$(date +%u)" == '7' ]]; then
mkdir -p "${backup_dir}/full/weekly/${ts}"
ln "${target_dir}/mysql-full.tgz.enc" "${backup_dir}/full/weekly/${ts}/mysql-full.tgz.enc"
cp "${target_dir}/mysql-full.tgz.enc.sha256" "${target_dir}/manifest.env" "${target_dir}/complete.env" "${backup_dir}/full/weekly/${ts}/"
upload_backup_set "${target_dir}" weekly "${ts}" "${server_uuid}" "${oss_uri}"
fi
if [[ "$(date +%d)" == '01' ]]; then
mkdir -p "${backup_dir}/full/monthly/${ts}"
ln "${target_dir}/mysql-full.tgz.enc" "${backup_dir}/full/monthly/${ts}/mysql-full.tgz.enc"
cp "${target_dir}/mysql-full.tgz.enc.sha256" "${target_dir}/manifest.env" "${target_dir}/complete.env" "${backup_dir}/full/monthly/${ts}/"
upload_backup_set "${target_dir}" monthly "${ts}" "${server_uuid}" "${oss_uri}"
fi
backup_log "全量热备完成：${target_dir}，位点 ${binlog_file}:${binlog_pos}"
}
prune_level() {
local base="$1"
local days="$2"
[[ -d "${base}" ]] || return
find "${base}" -mindepth 1 -maxdepth 1 -type d -mtime "+${days}" -print0 |
while IFS= read -r -d '' dir; do
backup_log "清理过期本地备份：${dir}"
rm -rf -- "${dir}"
done
}
prune_backups() {
local backup_dir daily weekly monthly
backup_dir="$(backup_require_env BACKUP_DIR)"
backup_validate_dir "${backup_dir}"
backup_lock_or_exit "${backup_dir}/.full.lock"
daily="$(backup_env_value BACKUP_KEEP_DAILY)"; daily="${daily:-14}"
weekly="$(backup_env_value BACKUP_KEEP_WEEKLY)"; weekly="${weekly:-8}"
monthly="$(backup_env_value BACKUP_KEEP_MONTHLY)"; monthly="${monthly:-12}"
[[ "${daily}" =~ ^[0-9]+$ && "${weekly}" =~ ^[0-9]+$ && "${monthly}" =~ ^[0-9]+$ ]] || {
backup_error 'BACKUP_KEEP_* 必须是非负整数'
exit 1
}
prune_level "${backup_dir}/full/daily" "${daily}"
prune_level "${backup_dir}/full/weekly" "$((weekly * 7))"
prune_level "${backup_dir}/full/monthly" "$((monthly * 31))"
}
status() {
local backup_dir oss_uri
backup_dir="$(backup_require_env BACKUP_DIR)"
oss_uri="$(backup_require_env BACKUP_OSS_URI)"
backup_validate_oss_uri "${oss_uri}"
backup_log '本地最近完整全量备份：'
find "${backup_dir}/full" -name complete.env -print 2>/dev/null | sort | tail -8 || true
backup_log 'OSS 最近完整全量备份：'
ossutil ls "${oss_uri}/mysql/full/" 2>/dev/null | grep 'complete.env' | tail -8 || true
backup_log 'OSS 最近 binlog 完成标识：'
ossutil ls "${oss_uri}/mysql/binlog/" -r 2>/dev/null | grep 'complete.env' | tail -8 || true
}
case "${1:-}" in
full) full_backup ;;
prune) prune_backups ;;
status) status ;;
*)
printf '用法: %s {full|prune|status}\n' "$0" >&2
exit 1
;;
esac
scripts/backup-prod.sh
-533
View File
@@ -1,533 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
COMPOSE_FILE="${ROOT_DIR}/deploy/docker-compose.prod.yml"
BACKEND_ENV="${ROOT_DIR}/backend/.env"
BACKUP_ROOT="${BACKUP_ROOT:-${ROOT_DIR}/backups}"
HELPER_IMAGE="${BACKUP_HELPER_IMAGE:-redis:7.4-alpine}"
if [[ "${BACKUP_ROOT}" != /* ]]; then
BACKUP_ROOT="${ROOT_DIR}/${BACKUP_ROOT}"
fi
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[1;34m'
NC='\033[0m'
log() {
printf "${BLUE}[backup]${NC} %s\n" "$*" >&2
}
log_success() {
printf "${GREEN}[backup]${NC} %s\n" "$*" >&2
}
log_warn() {
printf "${YELLOW}[backup]${NC} %s\n" "$*" >&2
}
log_error() {
printf "${RED}[backup]${NC} %s\n" "$*" >&2
}
need_cmd() {
if ! command -v "$1" >/dev/null 2>&1; then
log_error "缺少命令：$1"
exit 1
fi
}
absolute_dir() {
local dir="$1"
(
cd "${dir}"
pwd -P
)
}
show_help() {
cat << EOF
用法: ./scripts/backup-prod.sh <命令> [选项]
命令:
backup 创建完整生产备份
restore <备份目录> 从备份目录恢复生产数据（需要 --yes）
list 列出本机备份目录
选项:
--yes 确认执行破坏性恢复，只用于 restore
--no-stop backup 时不主动停止 backend/caddy，适合只读低风险窗口（不推荐）
-h, --help 显示帮助
环境变量:
BACKUP_ROOT=/path 备份根目录，默认 ./backups
BACKUP_HELPER_IMAGE=img 用于打包/还原 Docker volume 的镜像，默认 redis:7.4-alpine
备份内容:
- MySQL 逻辑备份 mysql.sql
- MinIO volume: minio_data.tgz
- Redis volume: redis_data.tgz
- Caddy 证书/配置 volume: caddy_data.tgz, caddy_config.tgz
- backend/.env、deploy 配置、backend/logs
- 当前 Git commit、Docker Compose 状态、goose 迁移状态
示例:
./scripts/backup-prod.sh backup
./scripts/backup-prod.sh list
./scripts/backup-prod.sh restore backups/20260618_153000 --yes
EOF
}
compose() {
local app_env caddy_domain caddy_email admin_base_path
app_env="$(env_value APP_ENV)"
caddy_domain="${CADDY_DOMAIN:-$(env_value CADDY_DOMAIN)}"
caddy_email="${CADDY_EMAIL:-$(env_value CADDY_EMAIL)}"
admin_base_path="${VITE_ADMIN_BASE_PATH:-$(env_value VITE_ADMIN_BASE_PATH)}"
if [[ "${app_env}" == "production" && (-z "${caddy_domain}" || -z "${caddy_email}") ]]; then
log_error "APP_ENV=production 时必须配置 CADDY_DOMAIN 和 CADDY_EMAIL"
exit 1
fi
CADDY_DOMAIN="${caddy_domain:-backup.localhost}" \
CADDY_EMAIL="${caddy_email:-backup@example.com}" \
VITE_ADMIN_BASE_PATH="${admin_base_path:-/manage-7x9k2p}" \
docker compose --env-file "${BACKEND_ENV}" -f "${COMPOSE_FILE}" "$@"
}
require_env_file() {
if [[ ! -f "${BACKEND_ENV}" ]]; then
log_error "未找到 backend/.env，无法读取生产环境变量"
exit 1
fi
}
env_value() {
local key="$1"
awk -F= -v key="${key}" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' "${BACKEND_ENV}"
}
require_env() {
local key="$1"
local value
value="$(env_value "${key}")"
if [[ -z "${value}" ]]; then
log_error "backend/.env 缺少或未填写 ${key}"
exit 1
fi
printf "%s" "${value}"
}
container_id() {
local service="$1"
local cid
cid="$(compose ps -a -q "${service}" 2>/dev/null || true)"
if [[ -z "${cid}" ]]; then
log "创建 ${service} 容器以挂载 volume..."
compose create "${service}" >/dev/null
cid="$(compose ps -a -q "${service}" 2>/dev/null || true)"
fi
if [[ -z "${cid}" ]]; then
log_error "无法找到 ${service} 容器"
exit 1
fi
printf "%s" "${cid}"
}
service_was_running() {
local service="$1"
local cid status
cid="$(compose ps -q "${service}" 2>/dev/null || true)"
if [[ -z "${cid}" ]]; then
return 1
fi
status="$(docker inspect -f '{{.State.Status}}' "${cid}" 2>/dev/null || true)"
[[ "${status}" == "running" ]]
}
start_services() {
if [[ "$#" -eq 0 ]]; then
return
fi
log "启动服务：$*"
compose up -d "$@"
}
stop_services() {
if [[ "$#" -eq 0 ]]; then
return
fi
log "停止服务：$*"
compose stop "$@"
}
start_recorded_services() {
if [[ "$#" -eq 0 ]]; then
return
fi
start_services "$@"
}
wait_service_ready() {
local service="$1"
local label="$2"
local timeout="${3:-120}"
local attempt cid status
log "等待 ${label} 就绪..."
for ((attempt = 1; attempt <= timeout; attempt++)); do
cid="$(compose ps -q "${service}" 2>/dev/null || true)"
if [[ -n "${cid}" ]]; then
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "${cid}" 2>/dev/null || true)"
if [[ "${status}" == "healthy" || "${status}" == "running" ]]; then
log_success "${label} 已就绪"
return
fi
fi
sleep 1
done
log_error "${label} 启动超时"
compose logs --tail=80 "${service}" || true
exit 1
}
remember_running() {
local service="$1"
local result_array_name="$2"
if service_was_running "${service}"; then
eval "${result_array_name}+=(\"${service}\")"
fi
}
tar_from_service_volume() {
local service="$1"
local source_dir="$2"
local backup_dir="$3"
local output_name="$4"
local cid
cid="$(container_id "${service}")"
log "备份 ${service}:${source_dir} -> ${output_name}"
docker run --rm \
--volumes-from "${cid}:ro" \
-v "${backup_dir}:/backup" \
"${HELPER_IMAGE}" \
sh -c "cd '${source_dir}' && tar czf '/backup/${output_name}' ."
}
restore_service_volume() {
local service="$1"
local target_dir="$2"
local backup_dir="$3"
local input_name="$4"
local cid
if [[ ! -f "${backup_dir}/${input_name}" ]]; then
log_warn "跳过 ${service}:${target_dir}，备份文件不存在：${input_name}"
return
fi
cid="$(container_id "${service}")"
log "还原 ${input_name} -> ${service}:${target_dir}"
docker run --rm \
--volumes-from "${cid}" \
-v "${backup_dir}:/backup:ro" \
"${HELPER_IMAGE}" \
sh -c "rm -rf '${target_dir}'/* '${target_dir}'/.[!.]* '${target_dir}'/..?* 2>/dev/null || true; tar xzf '/backup/${input_name}' -C '${target_dir}'"
}
copy_if_exists() {
local source="$1"
local target="$2"
if [[ -e "${source}" ]]; then
cp -a "${source}" "${target}"
fi
}
write_manifest() {
local backup_dir="$1"
{
printf "created_at=%s\n" "$(date -Iseconds)"
printf "host=%s\n" "$(hostname)"
printf "git_commit=%s\n" "$(git -C "${ROOT_DIR}" rev-parse HEAD 2>/dev/null || true)"
printf "git_branch=%s\n" "$(git -C "${ROOT_DIR}" branch --show-current 2>/dev/null || true)"
printf "mysql_database=%s\n" "$(require_env MYSQL_DATABASE)"
printf "storage_bucket=%s\n" "$(require_env STORAGE_BUCKET)"
} > "${backup_dir}/manifest.env"
git -C "${ROOT_DIR}" log --oneline -5 > "${backup_dir}/git-log.txt" 2>/dev/null || true
compose ps > "${backup_dir}/docker-compose-ps.txt" 2>/dev/null || true
compose config > "${backup_dir}/docker-compose.config.rendered.yml" 2>/dev/null || true
}
backup_mysql() {
local backup_dir="$1"
log "备份 MySQL -> mysql.sql"
compose exec -T mysql sh -c \
'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysqldump -uroot --single-transaction --routines --triggers --events --hex-blob --default-character-set=utf8mb4 "$MYSQL_DATABASE"' \
> "${backup_dir}/mysql.sql"
compose exec -T mysql sh -c 'mysql --version' > "${backup_dir}/mysql-version.txt" 2>/dev/null || true
}
backup_goose_status() {
local backup_dir="$1"
log "记录 goose 迁移状态"
compose run --rm --no-deps backend /app/goose -dir /app/migrations mysql "$(goose_dsn "$(require_env MYSQL_DSN)")" status \
> "${backup_dir}/goose-status.txt" 2>&1 || true
}
goose_dsn() {
local dsn="$1"
if [[ "${dsn}" != *"multiStatements="* ]]; then
if [[ "${dsn}" == *"?"* ]]; then
dsn="${dsn}&multiStatements=true"
else
dsn="${dsn}?multiStatements=true"
fi
fi
printf "%s" "${dsn}"
}
backup_files() {
local backup_dir="$1"
log "备份配置文件"
mkdir -p "${backup_dir}/config"
copy_if_exists "${BACKEND_ENV}" "${backup_dir}/config/backend.env"
copy_if_exists "${COMPOSE_FILE}" "${backup_dir}/config/docker-compose.prod.yml"
copy_if_exists "${ROOT_DIR}/deploy/caddy/Caddyfile" "${backup_dir}/config/Caddyfile"
copy_if_exists "${ROOT_DIR}/deploy/mysql/my.cnf" "${backup_dir}/config/mysql.my.cnf"
if [[ -d "${ROOT_DIR}/backend/logs" ]]; then
log "备份后端日志 -> backend_logs.tgz"
tar czf "${backup_dir}/backend_logs.tgz" -C "${ROOT_DIR}/backend" logs
fi
}
restore_files() {
local backup_dir="$1"
local safety_dir="${BACKUP_ROOT}/pre-restore-files-$(date +%Y%m%d_%H%M%S)"
mkdir -p "${safety_dir}"
log "恢复配置文件（当前文件会先备份到 ${safety_dir}）"
copy_if_exists "${BACKEND_ENV}" "${safety_dir}/backend.env"
copy_if_exists "${COMPOSE_FILE}" "${safety_dir}/docker-compose.prod.yml"
copy_if_exists "${ROOT_DIR}/deploy/caddy/Caddyfile" "${safety_dir}/Caddyfile"
copy_if_exists "${ROOT_DIR}/deploy/mysql/my.cnf" "${safety_dir}/mysql.my.cnf"
copy_if_exists "${backup_dir}/config/backend.env" "${BACKEND_ENV}"
copy_if_exists "${backup_dir}/config/docker-compose.prod.yml" "${COMPOSE_FILE}"
copy_if_exists "${backup_dir}/config/Caddyfile" "${ROOT_DIR}/deploy/caddy/Caddyfile"
copy_if_exists "${backup_dir}/config/mysql.my.cnf" "${ROOT_DIR}/deploy/mysql/my.cnf"
if [[ -f "${backup_dir}/backend_logs.tgz" ]]; then
log "恢复后端日志"
mkdir -p "${ROOT_DIR}/backend"
rm -rf "${ROOT_DIR}/backend/logs"
tar xzf "${backup_dir}/backend_logs.tgz" -C "${ROOT_DIR}/backend"
fi
}
prepare_restore_env() {
local backup_dir="$1"
if [[ ! -f "${BACKEND_ENV}" ]]; then
if [[ ! -f "${backup_dir}/config/backend.env" ]]; then
log_error "当前缺少 backend/.env，备份中也没有 config/backend.env，无法恢复"
exit 1
fi
log_warn "当前缺少 backend/.env，先从备份恢复环境变量文件"
mkdir -p "$(dirname "${BACKEND_ENV}")"
cp -a "${backup_dir}/config/backend.env" "${BACKEND_ENV}"
fi
if [[ ! -f "${COMPOSE_FILE}" && -f "${backup_dir}/config/docker-compose.prod.yml" ]]; then
log_warn "当前缺少 deploy/docker-compose.prod.yml，先从备份恢复 Compose 文件"
mkdir -p "$(dirname "${COMPOSE_FILE}")"
cp -a "${backup_dir}/config/docker-compose.prod.yml" "${COMPOSE_FILE}"
fi
require_env_file
}
restore_mysql() {
local backup_dir="$1"
local database
database="$(require_env MYSQL_DATABASE)"
if [[ ! "${database}" =~ ^[A-Za-z0-9_]+$ ]]; then
log_error "MYSQL_DATABASE 只能包含字母、数字和下划线，当前值：${database}"
exit 1
fi
if [[ ! -f "${backup_dir}/mysql.sql" ]]; then
log_error "缺少 MySQL 备份文件：${backup_dir}/mysql.sql"
exit 1
fi
start_services mysql
wait_service_ready mysql "MySQL"
log "重建数据库 ${database}"
compose exec -T mysql sh -c \
'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot -e "DROP DATABASE IF EXISTS \`$MYSQL_DATABASE\`; CREATE DATABASE \`$MYSQL_DATABASE\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"'
log "导入 MySQL 备份"
compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot "$MYSQL_DATABASE"' < "${backup_dir}/mysql.sql"
}
backup_all() {
require_env_file
need_cmd docker
local no_stop="$1"
local backup_dir="${BACKUP_ROOT}/$(date +%Y%m%d_%H%M%S)"
local restart_services=()
local backup_ok=0
backup_failure_cleanup() {
local exit_code=$?
trap - ERR
if [[ "${no_stop}" == "0" && "${backup_ok}" != "1" ]]; then
log_warn "备份失败，尝试恢复备份前运行的服务"
start_recorded_services "${restart_services[@]+"${restart_services[@]}"}" || true
fi
exit "${exit_code}"
}
trap backup_failure_cleanup ERR
umask 077
mkdir -p "${backup_dir}"
chmod 700 "${backup_dir}"
log "备份目录：${backup_dir}"
if [[ "${no_stop}" == "0" ]]; then
remember_running caddy restart_services
remember_running backend restart_services
stop_services caddy backend
else
log_warn "已开启 --no-stop，备份期间仍可能有写入，建议只在低风险窗口使用"
fi
start_services mysql
wait_service_ready mysql "MySQL"
backup_mysql "${backup_dir}"
backup_goose_status "${backup_dir}"
if [[ "${no_stop}" == "0" ]]; then
remember_running minio restart_services
remember_running redis restart_services
stop_services minio redis caddy
fi
tar_from_service_volume minio /data "${backup_dir}" minio_data.tgz
tar_from_service_volume redis /data "${backup_dir}" redis_data.tgz
tar_from_service_volume caddy /data "${backup_dir}" caddy_data.tgz
tar_from_service_volume caddy /config "${backup_dir}" caddy_config.tgz
backup_files "${backup_dir}"
write_manifest "${backup_dir}"
if [[ "${no_stop}" == "0" ]]; then
start_recorded_services "${restart_services[@]+"${restart_services[@]}"}"
fi
backup_ok=1
trap - ERR
log_success "备份完成：${backup_dir}"
log_warn "备份包含生产密钥，请离线加密保存，不要提交到 Git。"
}
restore_all() {
need_cmd docker
local backup_dir="$1"
local yes="$2"
local restore_ok=0
if [[ -z "${backup_dir}" || ! -d "${backup_dir}" ]]; then
log_error "备份目录不存在：${backup_dir}"
exit 1
fi
if [[ "${yes}" != "1" ]]; then
log_error "restore 会覆盖当前生产数据库、MinIO、Redis、Caddy volume 和配置文件。确认执行请追加 --yes"
exit 1
fi
backup_dir="$(absolute_dir "${backup_dir}")"
restore_failure_cleanup() {
local exit_code=$?
trap - ERR
if [[ "${restore_ok}" != "1" ]]; then
log_warn "恢复失败，尝试启动当前配置下的服务，便于继续排查"
compose up -d || true
fi
exit "${exit_code}"
}
trap restore_failure_cleanup ERR
prepare_restore_env "${backup_dir}"
log_warn "即将从 ${backup_dir} 恢复，当前生产数据会被覆盖。"
stop_services backend caddy minio redis
restore_files "${backup_dir}"
restore_mysql "${backup_dir}"
restore_service_volume minio /data "${backup_dir}" minio_data.tgz
restore_service_volume redis /data "${backup_dir}" redis_data.tgz
restore_service_volume caddy /data "${backup_dir}" caddy_data.tgz
restore_service_volume caddy /config "${backup_dir}" caddy_config.tgz
log "启动生产服务"
compose up -d
restore_ok=1
trap - ERR
log_success "恢复完成。建议立刻执行健康检查并抽查图片/二维码/订单数据。"
}
list_backups() {
if [[ ! -d "${BACKUP_ROOT}" ]]; then
log_warn "备份目录不存在：${BACKUP_ROOT}"
return
fi
find "${BACKUP_ROOT}" -mindepth 2 -maxdepth 2 -type f -name manifest.env -exec dirname {} \; | sort
}
main() {
local command="${1:-}"
local no_stop=0
local yes=0
local restore_dir=""
if [[ -z "${command}" || "${command}" == "-h" || "${command}" == "--help" ]]; then
show_help
exit 0
fi
shift || true
case "${command}" in
backup)
while [[ $# -gt 0 ]]; do
case "$1" in
--no-stop) no_stop=1; shift ;;
-h|--help) show_help; exit 0 ;;
*) log_error "未知选项：$1"; show_help; exit 1 ;;
esac
done
backup_all "${no_stop}"
;;
restore)
restore_dir="${1:-}"
if [[ $# -gt 0 ]]; then
shift
fi
while [[ $# -gt 0 ]]; do
case "$1" in
--yes) yes=1; shift ;;
-h|--help) show_help; exit 0 ;;
*) log_error "未知选项：$1"; show_help; exit 1 ;;
esac
done
restore_all "${restore_dir}" "${yes}"
;;
list)
list_backups
;;
*)
log_error "未知命令：${command}"
show_help
exit 1
;;
esac
}
main "$@"
scripts/deploy-prod.sh
+72 -14
View File
@@ -146,23 +146,28 @@ validate_env() {
exit 1
fi
local app_env caddy_domain caddy_email mysql_dsn redis_addr storage_endpoint backend_uid backend_gid
local minio_root_user minio_root_password storage_access_key_id storage_secret_access_key
local app_env caddy_domain caddy_email mysql_dsn redis_addr storage_endpoint backend_uid backend_gid backup_dir backup_restore_dir backup_oss_uri backup_user
app_env="$(require_env APP_ENV)"
caddy_domain="$(require_env CADDY_DOMAIN)"
caddy_email="$(require_env CADDY_EMAIL)"
mysql_dsn="$(require_env MYSQL_DSN)"
redis_addr="$(require_env REDIS_ADDR)"
storage_endpoint="$(require_env STORAGE_ENDPOINT)"
require_env MYSQL_DATABASE >/dev/null
local mysql_database
mysql_database="$(require_env MYSQL_DATABASE)"
require_env MYSQL_ROOT_PASSWORD >/dev/null
require_env MYSQL_USER >/dev/null
require_env MYSQL_PASSWORD >/dev/null
require_env JWT_SECRET >/dev/null
minio_root_user="$(require_env MINIO_ROOT_USER)"
minio_root_password="$(require_env MINIO_ROOT_PASSWORD)"
storage_access_key_id="$(require_env STORAGE_ACCESS_KEY_ID)"
storage_secret_access_key="$(require_env STORAGE_SECRET_ACCESS_KEY)"
require_env STORAGE_ACCESS_KEY_ID >/dev/null
require_env STORAGE_SECRET_ACCESS_KEY >/dev/null
backup_dir="$(require_env BACKUP_DIR)"
backup_restore_dir="$(require_env BACKUP_RESTORE_DIR)"
backup_oss_uri="$(require_env BACKUP_OSS_URI)"
require_env BACKUP_PASSPHRASE >/dev/null
require_env BACKUP_KMS_KEY_ID >/dev/null
backup_user="$(require_env BACKUP_MYSQL_USER)"
require_env BACKUP_MYSQL_PASSWORD >/dev/null
backend_uid="$(require_env BACKEND_UID)"
backend_gid="$(require_env BACKEND_GID)"
@@ -197,14 +202,36 @@ validate_env() {
exit 1
fi
if [[ "${storage_endpoint}" == *"127.0.0.1"* || "${storage_endpoint}" == *"localhost"* ]]; then
log_error "STORAGE_ENDPOINT 仍指向本机，请改为 http://minio:9000"
log_error "STORAGE_ENDPOINT 仍指向本机，请改为阿里云 OSS Endpoint（如 https://oss-cn-hangzhou.aliyuncs.com）"
exit 1
fi
if [[ "${storage_endpoint}" == "http://minio:9000" || "${storage_endpoint}" == "minio:9000" ]]; then
if [[ "${storage_access_key_id}" != "${minio_root_user}" || "${storage_secret_access_key}" != "${minio_root_password}" ]]; then
log_error "使用内置 MinIO 时，STORAGE_ACCESS_KEY_ID/SECRET_ACCESS_KEY 必须和 MINIO_ROOT_USER/ROOT_PASSWORD 保持一致"
exit 1
fi
if [[ "${storage_endpoint}" == *"minio"* ]]; then
log_error "STORAGE_ENDPOINT 仍指向已下线的 MinIO，请改为阿里云 OSS Endpoint"
exit 1
fi
if [[ ! "${storage_endpoint}" =~ ^https://oss-[A-Za-z0-9-]+\.aliyuncs\.com/?$ ]]; then
log_error "STORAGE_ENDPOINT 必须是阿里云 OSS HTTPS Endpoint，当前值：${storage_endpoint}"
exit 1
fi
if [[ "${backup_dir}" != /data/* || "${backup_dir}" == "/data" ]]; then
log_error "BACKUP_DIR 必须是 /data 之下的具体绝对路径，当前值：${backup_dir}"
exit 1
fi
if [[ "${backup_restore_dir}" != /data/* || "${backup_restore_dir}" == "/data" || "${backup_restore_dir}" == "${backup_dir}" ]]; then
log_error "BACKUP_RESTORE_DIR 必须是与 BACKUP_DIR 不同的 /data 子目录，当前值：${backup_restore_dir}"
exit 1
fi
if [[ ! "${backup_oss_uri}" =~ ^oss://[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
log_error "BACKUP_OSS_URI 必须是 oss://bucket 格式，当前值：${backup_oss_uri}"
exit 1
fi
if [[ ! "${backup_user}" =~ ^[A-Za-z0-9_]{1,32}$ ]]; then
log_error "BACKUP_MYSQL_USER 只能包含字母、数字和下划线，长度不超过 32"
exit 1
fi
if [[ ! "${mysql_database}" =~ ^[A-Za-z0-9_]+$ ]]; then
log_error "MYSQL_DATABASE 只能包含字母、数字和下划线，当前值：${mysql_database}"
exit 1
fi
if awk -F= '/^[A-Z0-9_]+=/{ if ($0 ~ /=(change-|保持你的|你的)/) found=1 } END { exit found ? 0 : 1 }' "${BACKEND_ENV}"; then
log_error "backend/.env 里还有占位值，请先替换后再部署"
@@ -407,6 +434,34 @@ mysql_root_exec() {
compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot "$@"' sh "$@"
}
sql_string_literal() {
local value="$1"
value="${value//\\/\\\\}"
value="${value//\'/\'\'}"
printf '%s' "${value}"
}
ensure_mysql_backup_user() {
local database backup_user backup_password escaped_password
database="$(require_env MYSQL_DATABASE)"
backup_user="$(require_env BACKUP_MYSQL_USER)"
backup_password="$(require_env BACKUP_MYSQL_PASSWORD)"
escaped_password="$(sql_string_literal "${backup_password}")"
unset backup_password
log "配置 MySQL 在线备份账号 ${backup_user}"
printf "CREATE USER IF NOT EXISTS \`%s\`@'%%' IDENTIFIED BY '%s';\n" "${backup_user}" "${escaped_password}" |
compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket'
printf "ALTER USER \`%s\`@'%%' IDENTIFIED BY '%s';\n" "${backup_user}" "${escaped_password}" |
compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket'
printf "GRANT BACKUP_ADMIN, PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO \`%s\`@'%%';\n" "${backup_user}" |
compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket'
printf "GRANT SELECT ON performance_schema.log_status TO \`%s\`@'%%';\n" "${backup_user}" |
compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket'
printf "GRANT SELECT ON \`%s\`.* TO \`%s\`@'%%';\n" "${database}" "${backup_user}" |
compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket'
}
goose_dsn() {
local dsn="$1"
if [[ "${dsn}" != *"multiStatements="* ]]; then
@@ -492,6 +547,9 @@ main() {
cd "${ROOT_DIR}"
need_cmd docker
need_cmd curl
need_cmd openssl
need_cmd ossutil
need_cmd flock
validate_env
export_caddy_env
prepare_log_dir
@@ -534,8 +592,8 @@ main() {
wait_service_healthy mysql "MySQL"
wait_service_healthy redis "Redis"
wait_service_healthy minio "MinIO"
wait_service_healthy caddy "Caddy"
ensure_mysql_backup_user
reset_database
run_migrations
scripts/restore-online.sh
Executable
+248
View File
@@ -0,0 +1,248 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
# 只用于隔离恢复演练或灾难恢复准备。它从不触碰生产 mysql 容器：prepare
# 在 BACKUP_RESTORE_DIR 下生成数据目录，start 仅启动 --network none 的临时容器。
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=backup-common.sh
source "${SCRIPT_DIR}/backup-common.sh"
field_value() {
local file="$1" key="$2"
awk -F= -v key="${key}" '$1 == key { print substr($0, index($0, "=") + 1); exit }' "${file}"
}
require_restore_dir() {
local dir
dir="$(backup_require_env BACKUP_RESTORE_DIR)"
backup_validate_dir "${dir}"
printf '%s' "${dir}"
}
validate_sha256() {
[[ "$1" =~ ^[0-9a-f]{64}$ ]] || { backup_error "非法 SHA-256：$1"; exit 1; }
}
validate_server_uuid() {
[[ "$1" =~ ^[0-9a-f-]{36}$ ]] || { backup_error "非法 server_uuid：$1"; exit 1; }
}
verify_downloaded_payload() {
local payload="$1" checksum_file="$2" complete_file="$3"
local expected actual expected_size actual_size
expected="$(field_value "${complete_file}" payload_sha256)"
expected_size="$(field_value "${complete_file}" payload_size_bytes)"
validate_sha256 "${expected}"
[[ "${expected_size}" =~ ^[0-9]+$ ]] || { backup_error '完成标识中的文件大小不合法'; exit 1; }
actual="$(backup_sha256 "${payload}")"
actual_size="$(backup_file_size "${payload}")"
[[ "${expected}" == "${actual}" && "${expected_size}" == "${actual_size}" ]] || {
backup_error "下载文件校验失败：${payload}"
exit 1
}
grep -Fqx "${expected} $(basename "${payload}")" "${checksum_file}" || {
backup_error "校验文件内容不匹配：${checksum_file}"
exit 1
}
}
prepare_full() {
backup_require_cmd docker
backup_require_cmd ossutil
backup_require_cmd openssl
local complete_uri="$1"
[[ "${complete_uri}" == oss://*/complete.env ]] || {
backup_error 'prepare 参数必须是完整全量备份的 complete.env OSS URI'
exit 1
}
local restore_dir remote_dir ts server_uuid work_dir download_dir
restore_dir="$(require_restore_dir)"
remote_dir="${complete_uri%/complete.env}"
ts="${remote_dir##*/}"
server_uuid="$(basename "$(dirname "${remote_dir}")")"
validate_server_uuid "${server_uuid}"
work_dir="${restore_dir}/${server_uuid}/${ts}"
download_dir="${work_dir}/.download"
[[ ! -e "${work_dir}" ]] || { backup_error "恢复目录已存在：${work_dir}"; exit 1; }
mkdir -p "${download_dir}" "${work_dir}/data"
trap 'rm -rf -- "${download_dir}"' EXIT
backup_oss_download "${complete_uri}" "${download_dir}/complete.env"
[[ "$(field_value "${download_dir}/complete.env" payload)" == 'mysql-full.tgz.enc' ]] || {
backup_error '不是受支持的全量备份完成标识'
exit 1
}
[[ "$(field_value "${download_dir}/complete.env" server_uuid)" == "${server_uuid}" ]] || {
backup_error '备份路径与完成标识的 server_uuid 不一致'
exit 1
}
backup_oss_download "${remote_dir}/mysql-full.tgz.enc" "${download_dir}/mysql-full.tgz.enc"
backup_oss_download "${remote_dir}/mysql-full.tgz.enc.sha256" "${download_dir}/mysql-full.tgz.enc.sha256"
verify_downloaded_payload "${download_dir}/mysql-full.tgz.enc" "${download_dir}/mysql-full.tgz.enc.sha256" "${download_dir}/complete.env"
backup_decrypt_file "${download_dir}/mysql-full.tgz.enc" "${download_dir}/mysql-full.tgz"
tar xzf "${download_dir}/mysql-full.tgz" -C "${work_dir}/data"
local manifest_uuid binlog_file binlog_pos
manifest_uuid="$(field_value "${work_dir}/data/manifest.env" server_uuid)"
binlog_file="$(field_value "${work_dir}/data/manifest.env" binlog_file)"
binlog_pos="$(field_value "${work_dir}/data/manifest.env" binlog_pos)"
[[ "${manifest_uuid}" == "${server_uuid}" ]] || { backup_error '归档内 manifest 与完成标识不一致'; exit 1; }
backup_binlog_index "${binlog_file}" >/dev/null
[[ "${binlog_pos}" =~ ^[0-9]+$ ]] || { backup_error 'manifest 中 binlog 位点非法'; exit 1; }
docker run --rm -v "${work_dir}/data:/restore" "${BACKUP_XTRABACKUP_IMAGE}" \
xtrabackup --prepare --target-dir=/restore
{
printf 'server_uuid=%s\n' "${server_uuid}"
printf 'binlog_file=%s\n' "${binlog_file}"
printf 'binlog_pos=%s\n' "${binlog_pos}"
printf 'source_complete_uri=%s\n' "${complete_uri}"
printf 'prepared_at=%s\n' "$(date -Iseconds)"
} > "${work_dir}/prepared.env"
rm -rf -- "${download_dir}"
trap - EXIT
backup_log "全量备份已 prepare：${work_dir}"
backup_log "下一步执行：$0 fetch-all ${work_dir}"
}
fetch_binlog() {
backup_require_cmd docker
backup_require_cmd ossutil
backup_require_cmd openssl
local prepared_dir="$1" complete_uri="$2"
[[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录：${prepared_dir}"; exit 1; }
[[ "${complete_uri}" == oss://*/complete.env ]] || { backup_error 'binlog 参数必须是 complete.env OSS URI'; exit 1; }
local remote_dir temp_dir complete source_file raw_sha payload server_uuid expected_uuid target_base
remote_dir="${complete_uri%/complete.env}"
temp_dir="${prepared_dir}/.binlog-download-$$"
mkdir -p "${temp_dir}" "${prepared_dir}/binlogs"
trap 'rm -rf -- "${temp_dir}"' EXIT
backup_oss_download "${complete_uri}" "${temp_dir}/complete.env"
source_file="$(field_value "${temp_dir}/complete.env" source_binlog)"
raw_sha="$(field_value "${temp_dir}/complete.env" raw_sha256)"
payload="$(field_value "${temp_dir}/complete.env" payload)"
server_uuid="$(field_value "${temp_dir}/complete.env" server_uuid)"
expected_uuid="$(field_value "${prepared_dir}/prepared.env" server_uuid)"
backup_binlog_index "${source_file}" >/dev/null
validate_sha256 "${raw_sha}"
validate_server_uuid "${server_uuid}"
[[ "${payload}" == 'binlog.enc' && "${server_uuid}" == "${expected_uuid}" ]] || {
backup_error 'binlog 完成标识与恢复目标不匹配'
exit 1
}
target_base="${prepared_dir}/binlogs/${source_file}.${raw_sha}"
if [[ -f "${target_base}.meta.env" && -f "${target_base}" ]]; then
rm -rf -- "${temp_dir}"
trap - EXIT
return
fi
backup_oss_download "${remote_dir}/binlog.enc" "${temp_dir}/binlog.enc"
backup_oss_download "${remote_dir}/binlog.enc.sha256" "${temp_dir}/binlog.enc.sha256"
verify_downloaded_payload "${temp_dir}/binlog.enc" "${temp_dir}/binlog.enc.sha256" "${temp_dir}/complete.env"
backup_decrypt_file "${temp_dir}/binlog.enc" "${target_base}"
docker run --rm -v "${prepared_dir}/binlogs:/binlogs:ro" mysql:8.4 \
mysqlbinlog --verify-binlog-checksum "/binlogs/$(basename "${target_base}")" >/dev/null
mv "${temp_dir}/complete.env" "${target_base}.meta.env"
rm -rf -- "${temp_dir}"
trap - EXIT
}
fetch_all() {
backup_require_cmd ossutil
local prepared_dir="$1" server_uuid oss_uri complete_uri
[[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录：${prepared_dir}"; exit 1; }
server_uuid="$(field_value "${prepared_dir}/prepared.env" server_uuid)"
oss_uri="$(backup_require_env BACKUP_OSS_URI)"
backup_validate_oss_uri "${oss_uri}"
while IFS= read -r complete_uri; do
[[ -n "${complete_uri}" ]] || continue
fetch_binlog "${prepared_dir}" "${complete_uri}"
done < <(ossutil ls "${oss_uri}/mysql/binlog/${server_uuid}/" -r | awk '$NF ~ /\/complete\.env$/ { print $NF }')
backup_log "已下载实例 ${server_uuid} 的全部完整 binlog"
}
start_isolated_mysql() {
backup_require_cmd docker
local prepared_dir="$1" name="${2:-}"
[[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录：${prepared_dir}"; exit 1; }
if [[ -z "${name}" ]]; then
name="hfb-restore-$(basename "${prepared_dir}")"
fi
[[ "${name}" =~ ^hfb-restore-[A-Za-z0-9_-]+$ ]] || { backup_error '恢复容器名必须以 hfb-restore- 开头'; exit 1; }
[[ -z "$(docker ps -aq -f "name=^/${name}$")" ]] || { backup_error "恢复容器已存在：${name}"; exit 1; }
docker run -d --name "${name}" --network none \
-e "MYSQL_ROOT_PASSWORD=$(backup_require_env MYSQL_ROOT_PASSWORD)" \
-v "${prepared_dir}/data:/var/lib/mysql" mysql:8.4 \
--skip-log-bin --skip-networking=1 >/dev/null
local attempt
for ((attempt = 1; attempt <= 60; attempt++)); do
if docker exec "${name}" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysqladmin -uroot --protocol=socket ping' >/dev/null 2>&1; then
backup_log "隔离 MySQL 已启动：${name}"
return
fi
sleep 1
done
backup_error "隔离 MySQL 启动失败，请执行 docker logs ${name}"
exit 1
}
replay_binlogs() {
backup_require_cmd docker
local prepared_dir="$1" target_container="$2" stop_datetime="${3:-}"
[[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录：${prepared_dir}"; exit 1; }
local production_cid
production_cid="$(backup_mysql_container_id)"
[[ "${target_container}" != "${production_cid}" ]] || { backup_error '禁止向生产 MySQL 回放 binlog'; exit 1; }
docker inspect "${target_container}" >/dev/null
[[ -d "${prepared_dir}/binlogs" ]] || { backup_error '尚未下载 binlog'; exit 1; }
local start_file start_pos metadata index source previous_index found=0
start_file="$(field_value "${prepared_dir}/prepared.env" binlog_file)"
start_pos="$(field_value "${prepared_dir}/prepared.env" binlog_pos)"
mapfile -t metadata < <(
for metadata in "${prepared_dir}"/binlogs/*.meta.env; do
[[ -f "${metadata}" ]] || continue
index="$(field_value "${metadata}" binlog_index)"
source="$(field_value "${metadata}" source_binlog)"
printf '%010d\t%s\t%s\n' "${index}" "${source}" "${metadata}"
done | sort -n
)
[[ "${#metadata[@]}" -gt 0 ]] || { backup_error '没有可回放的 binlog'; exit 1; }
local binlog_paths=()
previous_index=''
local row path
for row in "${metadata[@]}"; do
IFS=$'\t' read -r index source metadata <<< "${row}"
if [[ "${source}" == "${start_file}" ]]; then
found=1
fi
[[ "${found}" == '1' ]] || continue
if [[ -n "${previous_index}" && "${index}" -ne $((previous_index + 1)) ]]; then
backup_error "恢复 binlog 不连续：${previous_index} -> ${index}"
exit 1
fi
path="/binlogs/$(basename "${metadata%.meta.env}")"
binlog_paths+=("${path}")
previous_index="${index}"
done
[[ "${found}" == '1' ]] || { backup_error "缺少全量备份起始 binlog：${start_file}"; exit 1; }
local mysqlbinlog_args=(mysqlbinlog "--start-position=${start_pos}")
if [[ -n "${stop_datetime}" ]]; then
mysqlbinlog_args+=("--stop-datetime=${stop_datetime}")
fi
mysqlbinlog_args+=("${binlog_paths[@]}")
docker run --rm -i -v "${prepared_dir}/binlogs:/binlogs:ro" mysql:8.4 "${mysqlbinlog_args[@]}" |
docker exec -i "${target_container}" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket'
backup_log "binlog 已回放至隔离 MySQL：${target_container}"
}
case "${1:-}" in
prepare) [[ $# -eq 2 ]] || { backup_error '用法：prepare <full-complete-uri>'; exit 1; }; prepare_full "$2" ;;
fetch-all) [[ $# -eq 2 ]] || { backup_error '用法：fetch-all <prepared-dir>'; exit 1; }; fetch_all "$2" ;;
fetch-binlog) [[ $# -eq 3 ]] || { backup_error '用法：fetch-binlog <prepared-dir> <binlog-complete-uri>'; exit 1; }; fetch_binlog "$2" "$3" ;;
start) [[ $# -le 3 && $# -ge 2 ]] || { backup_error '用法：start <prepared-dir> [container-name]'; exit 1; }; start_isolated_mysql "$2" "${3:-}" ;;
replay) [[ $# -ge 3 && $# -le 4 ]] || { backup_error '用法：replay <prepared-dir> <container> [YYYY-MM-DD HH:MM:SS]'; exit 1; }; replay_binlogs "$2" "$3" "${4:-}" ;;
*)
printf '用法: %s {prepare|fetch-all|fetch-binlog|start|replay} ...\n' "$0" >&2
exit 1
;;
esac