249 lines
12 KiB
Bash
Executable File
249 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
umask 077
|
||
|
||
# 只用于隔离恢复演练或灾难恢复准备。它从不触碰生产 mysql 容器：prepare
|
||
# 在 BACKUP_RESTORE_DIR 下生成数据目录，start 仅启动 --network none 的临时容器。
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
# shellcheck source=backup-common.sh
|
||
source "${SCRIPT_DIR}/backup-common.sh"
|
||
|
||
field_value() {
|
||
local file="$1" key="$2"
|
||
awk -F= -v key="${key}" '$1 == key { print substr($0, index($0, "=") + 1); exit }' "${file}"
|
||
}
|
||
|
||
require_restore_dir() {
|
||
local dir
|
||
dir="$(backup_require_env BACKUP_RESTORE_DIR)"
|
||
backup_validate_dir "${dir}"
|
||
printf '%s' "${dir}"
|
||
}
|
||
|
||
validate_sha256() {
|
||
[[ "$1" =~ ^[0-9a-f]{64}$ ]] || { backup_error "非法 SHA-256：$1"; exit 1; }
|
||
}
|
||
|
||
validate_server_uuid() {
|
||
[[ "$1" =~ ^[0-9a-f-]{36}$ ]] || { backup_error "非法 server_uuid：$1"; exit 1; }
|
||
}
|
||
|
||
verify_downloaded_payload() {
|
||
local payload="$1" checksum_file="$2" complete_file="$3"
|
||
local expected actual expected_size actual_size
|
||
expected="$(field_value "${complete_file}" payload_sha256)"
|
||
expected_size="$(field_value "${complete_file}" payload_size_bytes)"
|
||
validate_sha256 "${expected}"
|
||
[[ "${expected_size}" =~ ^[0-9]+$ ]] || { backup_error '完成标识中的文件大小不合法'; exit 1; }
|
||
actual="$(backup_sha256 "${payload}")"
|
||
actual_size="$(backup_file_size "${payload}")"
|
||
[[ "${expected}" == "${actual}" && "${expected_size}" == "${actual_size}" ]] || {
|
||
backup_error "下载文件校验失败：${payload}"
|
||
exit 1
|
||
}
|
||
grep -Fqx "${expected} $(basename "${payload}")" "${checksum_file}" || {
|
||
backup_error "校验文件内容不匹配：${checksum_file}"
|
||
exit 1
|
||
}
|
||
}
|
||
|
||
prepare_full() {
|
||
backup_require_cmd docker
|
||
backup_require_cmd ossutil
|
||
backup_require_cmd openssl
|
||
local complete_uri="$1"
|
||
[[ "${complete_uri}" == oss://*/complete.env ]] || {
|
||
backup_error 'prepare 参数必须是完整全量备份的 complete.env OSS URI'
|
||
exit 1
|
||
}
|
||
local restore_dir remote_dir ts server_uuid work_dir download_dir
|
||
restore_dir="$(require_restore_dir)"
|
||
remote_dir="${complete_uri%/complete.env}"
|
||
ts="${remote_dir##*/}"
|
||
server_uuid="$(basename "$(dirname "${remote_dir}")")"
|
||
validate_server_uuid "${server_uuid}"
|
||
work_dir="${restore_dir}/${server_uuid}/${ts}"
|
||
download_dir="${work_dir}/.download"
|
||
[[ ! -e "${work_dir}" ]] || { backup_error "恢复目录已存在：${work_dir}"; exit 1; }
|
||
mkdir -p "${download_dir}" "${work_dir}/data"
|
||
trap 'rm -rf -- "${download_dir}"' EXIT
|
||
|
||
backup_oss_download "${complete_uri}" "${download_dir}/complete.env"
|
||
[[ "$(field_value "${download_dir}/complete.env" payload)" == 'mysql-full.tgz.enc' ]] || {
|
||
backup_error '不是受支持的全量备份完成标识'
|
||
exit 1
|
||
}
|
||
[[ "$(field_value "${download_dir}/complete.env" server_uuid)" == "${server_uuid}" ]] || {
|
||
backup_error '备份路径与完成标识的 server_uuid 不一致'
|
||
exit 1
|
||
}
|
||
backup_oss_download "${remote_dir}/mysql-full.tgz.enc" "${download_dir}/mysql-full.tgz.enc"
|
||
backup_oss_download "${remote_dir}/mysql-full.tgz.enc.sha256" "${download_dir}/mysql-full.tgz.enc.sha256"
|
||
verify_downloaded_payload "${download_dir}/mysql-full.tgz.enc" "${download_dir}/mysql-full.tgz.enc.sha256" "${download_dir}/complete.env"
|
||
backup_decrypt_file "${download_dir}/mysql-full.tgz.enc" "${download_dir}/mysql-full.tgz"
|
||
tar xzf "${download_dir}/mysql-full.tgz" -C "${work_dir}/data"
|
||
local manifest_uuid binlog_file binlog_pos
|
||
manifest_uuid="$(field_value "${work_dir}/data/manifest.env" server_uuid)"
|
||
binlog_file="$(field_value "${work_dir}/data/manifest.env" binlog_file)"
|
||
binlog_pos="$(field_value "${work_dir}/data/manifest.env" binlog_pos)"
|
||
[[ "${manifest_uuid}" == "${server_uuid}" ]] || { backup_error '归档内 manifest 与完成标识不一致'; exit 1; }
|
||
backup_binlog_index "${binlog_file}" >/dev/null
|
||
[[ "${binlog_pos}" =~ ^[0-9]+$ ]] || { backup_error 'manifest 中 binlog 位点非法'; exit 1; }
|
||
docker run --rm -v "${work_dir}/data:/restore" "${BACKUP_XTRABACKUP_IMAGE}" \
|
||
xtrabackup --prepare --target-dir=/restore
|
||
{
|
||
printf 'server_uuid=%s\n' "${server_uuid}"
|
||
printf 'binlog_file=%s\n' "${binlog_file}"
|
||
printf 'binlog_pos=%s\n' "${binlog_pos}"
|
||
printf 'source_complete_uri=%s\n' "${complete_uri}"
|
||
printf 'prepared_at=%s\n' "$(date -Iseconds)"
|
||
} > "${work_dir}/prepared.env"
|
||
rm -rf -- "${download_dir}"
|
||
trap - EXIT
|
||
backup_log "全量备份已 prepare：${work_dir}"
|
||
backup_log "下一步执行：$0 fetch-all ${work_dir}"
|
||
}
|
||
|
||
fetch_binlog() {
|
||
backup_require_cmd docker
|
||
backup_require_cmd ossutil
|
||
backup_require_cmd openssl
|
||
local prepared_dir="$1" complete_uri="$2"
|
||
[[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录：${prepared_dir}"; exit 1; }
|
||
[[ "${complete_uri}" == oss://*/complete.env ]] || { backup_error 'binlog 参数必须是 complete.env OSS URI'; exit 1; }
|
||
local remote_dir temp_dir complete source_file raw_sha payload server_uuid expected_uuid target_base
|
||
remote_dir="${complete_uri%/complete.env}"
|
||
temp_dir="${prepared_dir}/.binlog-download-$$"
|
||
mkdir -p "${temp_dir}" "${prepared_dir}/binlogs"
|
||
trap 'rm -rf -- "${temp_dir}"' EXIT
|
||
backup_oss_download "${complete_uri}" "${temp_dir}/complete.env"
|
||
source_file="$(field_value "${temp_dir}/complete.env" source_binlog)"
|
||
raw_sha="$(field_value "${temp_dir}/complete.env" raw_sha256)"
|
||
payload="$(field_value "${temp_dir}/complete.env" payload)"
|
||
server_uuid="$(field_value "${temp_dir}/complete.env" server_uuid)"
|
||
expected_uuid="$(field_value "${prepared_dir}/prepared.env" server_uuid)"
|
||
backup_binlog_index "${source_file}" >/dev/null
|
||
validate_sha256 "${raw_sha}"
|
||
validate_server_uuid "${server_uuid}"
|
||
[[ "${payload}" == 'binlog.enc' && "${server_uuid}" == "${expected_uuid}" ]] || {
|
||
backup_error 'binlog 完成标识与恢复目标不匹配'
|
||
exit 1
|
||
}
|
||
target_base="${prepared_dir}/binlogs/${source_file}.${raw_sha}"
|
||
if [[ -f "${target_base}.meta.env" && -f "${target_base}" ]]; then
|
||
rm -rf -- "${temp_dir}"
|
||
trap - EXIT
|
||
return
|
||
fi
|
||
backup_oss_download "${remote_dir}/binlog.enc" "${temp_dir}/binlog.enc"
|
||
backup_oss_download "${remote_dir}/binlog.enc.sha256" "${temp_dir}/binlog.enc.sha256"
|
||
verify_downloaded_payload "${temp_dir}/binlog.enc" "${temp_dir}/binlog.enc.sha256" "${temp_dir}/complete.env"
|
||
backup_decrypt_file "${temp_dir}/binlog.enc" "${target_base}"
|
||
docker run --rm -v "${prepared_dir}/binlogs:/binlogs:ro" mysql:8.4 \
|
||
mysqlbinlog --verify-binlog-checksum "/binlogs/$(basename "${target_base}")" >/dev/null
|
||
mv "${temp_dir}/complete.env" "${target_base}.meta.env"
|
||
rm -rf -- "${temp_dir}"
|
||
trap - EXIT
|
||
}
|
||
|
||
fetch_all() {
|
||
backup_require_cmd ossutil
|
||
local prepared_dir="$1" server_uuid oss_uri complete_uri
|
||
[[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录：${prepared_dir}"; exit 1; }
|
||
server_uuid="$(field_value "${prepared_dir}/prepared.env" server_uuid)"
|
||
oss_uri="$(backup_require_env BACKUP_OSS_URI)"
|
||
backup_validate_oss_uri "${oss_uri}"
|
||
while IFS= read -r complete_uri; do
|
||
[[ -n "${complete_uri}" ]] || continue
|
||
fetch_binlog "${prepared_dir}" "${complete_uri}"
|
||
done < <(ossutil ls "${oss_uri}/mysql/binlog/${server_uuid}/" -r | awk '$NF ~ /\/complete\.env$/ { print $NF }')
|
||
backup_log "已下载实例 ${server_uuid} 的全部完整 binlog"
|
||
}
|
||
|
||
start_isolated_mysql() {
|
||
backup_require_cmd docker
|
||
local prepared_dir="$1" name="${2:-}"
|
||
[[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录：${prepared_dir}"; exit 1; }
|
||
if [[ -z "${name}" ]]; then
|
||
name="hfb-restore-$(basename "${prepared_dir}")"
|
||
fi
|
||
[[ "${name}" =~ ^hfb-restore-[A-Za-z0-9_-]+$ ]] || { backup_error '恢复容器名必须以 hfb-restore- 开头'; exit 1; }
|
||
[[ -z "$(docker ps -aq -f "name=^/${name}$")" ]] || { backup_error "恢复容器已存在：${name}"; exit 1; }
|
||
docker run -d --name "${name}" --network none \
|
||
-e "MYSQL_ROOT_PASSWORD=$(backup_require_env MYSQL_ROOT_PASSWORD)" \
|
||
-v "${prepared_dir}/data:/var/lib/mysql" mysql:8.4 \
|
||
--skip-log-bin --skip-networking=1 >/dev/null
|
||
local attempt
|
||
for ((attempt = 1; attempt <= 60; attempt++)); do
|
||
if docker exec "${name}" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysqladmin -uroot --protocol=socket ping' >/dev/null 2>&1; then
|
||
backup_log "隔离 MySQL 已启动：${name}"
|
||
return
|
||
fi
|
||
sleep 1
|
||
done
|
||
backup_error "隔离 MySQL 启动失败，请执行 docker logs ${name}"
|
||
exit 1
|
||
}
|
||
|
||
replay_binlogs() {
|
||
backup_require_cmd docker
|
||
local prepared_dir="$1" target_container="$2" stop_datetime="${3:-}"
|
||
[[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录：${prepared_dir}"; exit 1; }
|
||
local production_cid
|
||
production_cid="$(backup_mysql_container_id)"
|
||
[[ "${target_container}" != "${production_cid}" ]] || { backup_error '禁止向生产 MySQL 回放 binlog'; exit 1; }
|
||
docker inspect "${target_container}" >/dev/null
|
||
[[ -d "${prepared_dir}/binlogs" ]] || { backup_error '尚未下载 binlog'; exit 1; }
|
||
local start_file start_pos metadata index source previous_index found=0
|
||
start_file="$(field_value "${prepared_dir}/prepared.env" binlog_file)"
|
||
start_pos="$(field_value "${prepared_dir}/prepared.env" binlog_pos)"
|
||
mapfile -t metadata < <(
|
||
for metadata in "${prepared_dir}"/binlogs/*.meta.env; do
|
||
[[ -f "${metadata}" ]] || continue
|
||
index="$(field_value "${metadata}" binlog_index)"
|
||
source="$(field_value "${metadata}" source_binlog)"
|
||
printf '%010d\t%s\t%s\n' "${index}" "${source}" "${metadata}"
|
||
done | sort -n
|
||
)
|
||
[[ "${#metadata[@]}" -gt 0 ]] || { backup_error '没有可回放的 binlog'; exit 1; }
|
||
local binlog_paths=()
|
||
previous_index=''
|
||
local row path
|
||
for row in "${metadata[@]}"; do
|
||
IFS=$'\t' read -r index source metadata <<< "${row}"
|
||
if [[ "${source}" == "${start_file}" ]]; then
|
||
found=1
|
||
fi
|
||
[[ "${found}" == '1' ]] || continue
|
||
if [[ -n "${previous_index}" && "${index}" -ne $((previous_index + 1)) ]]; then
|
||
backup_error "恢复 binlog 不连续：${previous_index} -> ${index}"
|
||
exit 1
|
||
fi
|
||
path="/binlogs/$(basename "${metadata%.meta.env}")"
|
||
binlog_paths+=("${path}")
|
||
previous_index="${index}"
|
||
done
|
||
[[ "${found}" == '1' ]] || { backup_error "缺少全量备份起始 binlog：${start_file}"; exit 1; }
|
||
local mysqlbinlog_args=(mysqlbinlog "--start-position=${start_pos}")
|
||
if [[ -n "${stop_datetime}" ]]; then
|
||
mysqlbinlog_args+=("--stop-datetime=${stop_datetime}")
|
||
fi
|
||
mysqlbinlog_args+=("${binlog_paths[@]}")
|
||
docker run --rm -i -v "${prepared_dir}/binlogs:/binlogs:ro" mysql:8.4 "${mysqlbinlog_args[@]}" |
|
||
docker exec -i "${target_container}" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket'
|
||
backup_log "binlog 已回放至隔离 MySQL：${target_container}"
|
||
}
|
||
|
||
case "${1:-}" in
|
||
prepare) [[ $# -eq 2 ]] || { backup_error '用法：prepare <full-complete-uri>'; exit 1; }; prepare_full "$2" ;;
|
||
fetch-all) [[ $# -eq 2 ]] || { backup_error '用法：fetch-all <prepared-dir>'; exit 1; }; fetch_all "$2" ;;
|
||
fetch-binlog) [[ $# -eq 3 ]] || { backup_error '用法：fetch-binlog <prepared-dir> <binlog-complete-uri>'; exit 1; }; fetch_binlog "$2" "$3" ;;
|
||
start) [[ $# -le 3 && $# -ge 2 ]] || { backup_error '用法：start <prepared-dir> [container-name]'; exit 1; }; start_isolated_mysql "$2" "${3:-}" ;;
|
||
replay) [[ $# -ge 3 && $# -le 4 ]] || { backup_error '用法：replay <prepared-dir> <container> [YYYY-MM-DD HH:MM:SS]'; exit 1; }; replay_binlogs "$2" "$3" "${4:-}" ;;
|
||
*)
|
||
printf '用法: %s {prepare|fetch-all|fetch-binlog|start|replay} ...\n' "$0" >&2
|
||
exit 1
|
||
;;
|
||
esac
|