#!/usr/bin/env bash set -euo pipefail umask 077 # 只用于隔离恢复演练或灾难恢复准备。它从不触碰生产 mysql 容器:prepare # 在 BACKUP_RESTORE_DIR 下生成数据目录,start 仅启动 --network none 的临时容器。 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=backup-common.sh source "${SCRIPT_DIR}/backup-common.sh" field_value() { local file="$1" key="$2" awk -F= -v key="${key}" '$1 == key { print substr($0, index($0, "=") + 1); exit }' "${file}" } require_restore_dir() { local dir dir="$(backup_require_env BACKUP_RESTORE_DIR)" backup_validate_dir "${dir}" printf '%s' "${dir}" } validate_sha256() { [[ "$1" =~ ^[0-9a-f]{64}$ ]] || { backup_error "非法 SHA-256:$1"; exit 1; } } validate_server_uuid() { [[ "$1" =~ ^[0-9a-f-]{36}$ ]] || { backup_error "非法 server_uuid:$1"; exit 1; } } verify_downloaded_payload() { local payload="$1" checksum_file="$2" complete_file="$3" local expected actual expected_size actual_size expected="$(field_value "${complete_file}" payload_sha256)" expected_size="$(field_value "${complete_file}" payload_size_bytes)" validate_sha256 "${expected}" [[ "${expected_size}" =~ ^[0-9]+$ ]] || { backup_error '完成标识中的文件大小不合法'; exit 1; } actual="$(backup_sha256 "${payload}")" actual_size="$(backup_file_size "${payload}")" [[ "${expected}" == "${actual}" && "${expected_size}" == "${actual_size}" ]] || { backup_error "下载文件校验失败:${payload}" exit 1 } grep -Fqx "${expected} $(basename "${payload}")" "${checksum_file}" || { backup_error "校验文件内容不匹配:${checksum_file}" exit 1 } } prepare_full() { backup_require_cmd docker backup_require_cmd ossutil backup_require_cmd openssl local complete_uri="$1" [[ "${complete_uri}" == oss://*/complete.env ]] || { backup_error 'prepare 参数必须是完整全量备份的 complete.env OSS URI' exit 1 } local restore_dir remote_dir ts server_uuid work_dir download_dir restore_dir="$(require_restore_dir)" remote_dir="${complete_uri%/complete.env}" ts="${remote_dir##*/}" server_uuid="$(basename "$(dirname "${remote_dir}")")" validate_server_uuid "${server_uuid}" work_dir="${restore_dir}/${server_uuid}/${ts}" download_dir="${work_dir}/.download" [[ ! -e "${work_dir}" ]] || { backup_error "恢复目录已存在:${work_dir}"; exit 1; } mkdir -p "${download_dir}" "${work_dir}/data" trap 'rm -rf -- "${download_dir}"' EXIT backup_oss_download "${complete_uri}" "${download_dir}/complete.env" [[ "$(field_value "${download_dir}/complete.env" payload)" == 'mysql-full.tgz.enc' ]] || { backup_error '不是受支持的全量备份完成标识' exit 1 } [[ "$(field_value "${download_dir}/complete.env" server_uuid)" == "${server_uuid}" ]] || { backup_error '备份路径与完成标识的 server_uuid 不一致' exit 1 } backup_oss_download "${remote_dir}/mysql-full.tgz.enc" "${download_dir}/mysql-full.tgz.enc" backup_oss_download "${remote_dir}/mysql-full.tgz.enc.sha256" "${download_dir}/mysql-full.tgz.enc.sha256" verify_downloaded_payload "${download_dir}/mysql-full.tgz.enc" "${download_dir}/mysql-full.tgz.enc.sha256" "${download_dir}/complete.env" backup_decrypt_file "${download_dir}/mysql-full.tgz.enc" "${download_dir}/mysql-full.tgz" tar xzf "${download_dir}/mysql-full.tgz" -C "${work_dir}/data" local manifest_uuid binlog_file binlog_pos manifest_uuid="$(field_value "${work_dir}/data/manifest.env" server_uuid)" binlog_file="$(field_value "${work_dir}/data/manifest.env" binlog_file)" binlog_pos="$(field_value "${work_dir}/data/manifest.env" binlog_pos)" [[ "${manifest_uuid}" == "${server_uuid}" ]] || { backup_error '归档内 manifest 与完成标识不一致'; exit 1; } backup_binlog_index "${binlog_file}" >/dev/null [[ "${binlog_pos}" =~ ^[0-9]+$ ]] || { backup_error 'manifest 中 binlog 位点非法'; exit 1; } docker run --rm -v "${work_dir}/data:/restore" "${BACKUP_XTRABACKUP_IMAGE}" \ xtrabackup --prepare --target-dir=/restore { printf 'server_uuid=%s\n' "${server_uuid}" printf 'binlog_file=%s\n' "${binlog_file}" printf 'binlog_pos=%s\n' "${binlog_pos}" printf 'source_complete_uri=%s\n' "${complete_uri}" printf 'prepared_at=%s\n' "$(date -Iseconds)" } > "${work_dir}/prepared.env" rm -rf -- "${download_dir}" trap - EXIT backup_log "全量备份已 prepare:${work_dir}" backup_log "下一步执行:$0 fetch-all ${work_dir}" } fetch_binlog() { backup_require_cmd docker backup_require_cmd ossutil backup_require_cmd openssl local prepared_dir="$1" complete_uri="$2" [[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录:${prepared_dir}"; exit 1; } [[ "${complete_uri}" == oss://*/complete.env ]] || { backup_error 'binlog 参数必须是 complete.env OSS URI'; exit 1; } local remote_dir temp_dir complete source_file raw_sha payload server_uuid expected_uuid target_base remote_dir="${complete_uri%/complete.env}" temp_dir="${prepared_dir}/.binlog-download-$$" mkdir -p "${temp_dir}" "${prepared_dir}/binlogs" trap 'rm -rf -- "${temp_dir}"' EXIT backup_oss_download "${complete_uri}" "${temp_dir}/complete.env" source_file="$(field_value "${temp_dir}/complete.env" source_binlog)" raw_sha="$(field_value "${temp_dir}/complete.env" raw_sha256)" payload="$(field_value "${temp_dir}/complete.env" payload)" server_uuid="$(field_value "${temp_dir}/complete.env" server_uuid)" expected_uuid="$(field_value "${prepared_dir}/prepared.env" server_uuid)" backup_binlog_index "${source_file}" >/dev/null validate_sha256 "${raw_sha}" validate_server_uuid "${server_uuid}" [[ "${payload}" == 'binlog.enc' && "${server_uuid}" == "${expected_uuid}" ]] || { backup_error 'binlog 完成标识与恢复目标不匹配' exit 1 } target_base="${prepared_dir}/binlogs/${source_file}.${raw_sha}" if [[ -f "${target_base}.meta.env" && -f "${target_base}" ]]; then rm -rf -- "${temp_dir}" trap - EXIT return fi backup_oss_download "${remote_dir}/binlog.enc" "${temp_dir}/binlog.enc" backup_oss_download "${remote_dir}/binlog.enc.sha256" "${temp_dir}/binlog.enc.sha256" verify_downloaded_payload "${temp_dir}/binlog.enc" "${temp_dir}/binlog.enc.sha256" "${temp_dir}/complete.env" backup_decrypt_file "${temp_dir}/binlog.enc" "${target_base}" docker run --rm -v "${prepared_dir}/binlogs:/binlogs:ro" mysql:8.4 \ mysqlbinlog --verify-binlog-checksum "/binlogs/$(basename "${target_base}")" >/dev/null mv "${temp_dir}/complete.env" "${target_base}.meta.env" rm -rf -- "${temp_dir}" trap - EXIT } fetch_all() { backup_require_cmd ossutil local prepared_dir="$1" server_uuid oss_uri complete_uri [[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录:${prepared_dir}"; exit 1; } server_uuid="$(field_value "${prepared_dir}/prepared.env" server_uuid)" oss_uri="$(backup_require_env BACKUP_OSS_URI)" backup_validate_oss_uri "${oss_uri}" while IFS= read -r complete_uri; do [[ -n "${complete_uri}" ]] || continue fetch_binlog "${prepared_dir}" "${complete_uri}" done < <(ossutil ls "${oss_uri}/mysql/binlog/${server_uuid}/" -r | awk '$NF ~ /\/complete\.env$/ { print $NF }') backup_log "已下载实例 ${server_uuid} 的全部完整 binlog" } start_isolated_mysql() { backup_require_cmd docker local prepared_dir="$1" name="${2:-}" [[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录:${prepared_dir}"; exit 1; } if [[ -z "${name}" ]]; then name="hfb-restore-$(basename "${prepared_dir}")" fi [[ "${name}" =~ ^hfb-restore-[A-Za-z0-9_-]+$ ]] || { backup_error '恢复容器名必须以 hfb-restore- 开头'; exit 1; } [[ -z "$(docker ps -aq -f "name=^/${name}$")" ]] || { backup_error "恢复容器已存在:${name}"; exit 1; } docker run -d --name "${name}" --network none \ -e "MYSQL_ROOT_PASSWORD=$(backup_require_env MYSQL_ROOT_PASSWORD)" \ -v "${prepared_dir}/data:/var/lib/mysql" mysql:8.4 \ --skip-log-bin --skip-networking=1 >/dev/null local attempt for ((attempt = 1; attempt <= 60; attempt++)); do if docker exec "${name}" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysqladmin -uroot --protocol=socket ping' >/dev/null 2>&1; then backup_log "隔离 MySQL 已启动:${name}" return fi sleep 1 done backup_error "隔离 MySQL 启动失败,请执行 docker logs ${name}" exit 1 } replay_binlogs() { backup_require_cmd docker local prepared_dir="$1" target_container="$2" stop_datetime="${3:-}" [[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录:${prepared_dir}"; exit 1; } local production_cid production_cid="$(backup_mysql_container_id)" [[ "${target_container}" != "${production_cid}" ]] || { backup_error '禁止向生产 MySQL 回放 binlog'; exit 1; } docker inspect "${target_container}" >/dev/null [[ -d "${prepared_dir}/binlogs" ]] || { backup_error '尚未下载 binlog'; exit 1; } local start_file start_pos metadata index source previous_index found=0 start_file="$(field_value "${prepared_dir}/prepared.env" binlog_file)" start_pos="$(field_value "${prepared_dir}/prepared.env" binlog_pos)" mapfile -t metadata < <( for metadata in "${prepared_dir}"/binlogs/*.meta.env; do [[ -f "${metadata}" ]] || continue index="$(field_value "${metadata}" binlog_index)" source="$(field_value "${metadata}" source_binlog)" printf '%010d\t%s\t%s\n' "${index}" "${source}" "${metadata}" done | sort -n ) [[ "${#metadata[@]}" -gt 0 ]] || { backup_error '没有可回放的 binlog'; exit 1; } local binlog_paths=() previous_index='' local row path for row in "${metadata[@]}"; do IFS=$'\t' read -r index source metadata <<< "${row}" if [[ "${source}" == "${start_file}" ]]; then found=1 fi [[ "${found}" == '1' ]] || continue if [[ -n "${previous_index}" && "${index}" -ne $((previous_index + 1)) ]]; then backup_error "恢复 binlog 不连续:${previous_index} -> ${index}" exit 1 fi path="/binlogs/$(basename "${metadata%.meta.env}")" binlog_paths+=("${path}") previous_index="${index}" done [[ "${found}" == '1' ]] || { backup_error "缺少全量备份起始 binlog:${start_file}"; exit 1; } local mysqlbinlog_args=(mysqlbinlog "--start-position=${start_pos}") if [[ -n "${stop_datetime}" ]]; then mysqlbinlog_args+=("--stop-datetime=${stop_datetime}") fi mysqlbinlog_args+=("${binlog_paths[@]}") docker run --rm -i -v "${prepared_dir}/binlogs:/binlogs:ro" mysql:8.4 "${mysqlbinlog_args[@]}" | docker exec -i "${target_container}" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket' backup_log "binlog 已回放至隔离 MySQL:${target_container}" } case "${1:-}" in prepare) [[ $# -eq 2 ]] || { backup_error '用法:prepare '; exit 1; }; prepare_full "$2" ;; fetch-all) [[ $# -eq 2 ]] || { backup_error '用法:fetch-all '; exit 1; }; fetch_all "$2" ;; fetch-binlog) [[ $# -eq 3 ]] || { backup_error '用法:fetch-binlog '; exit 1; }; fetch_binlog "$2" "$3" ;; start) [[ $# -le 3 && $# -ge 2 ]] || { backup_error '用法:start [container-name]'; exit 1; }; start_isolated_mysql "$2" "${3:-}" ;; replay) [[ $# -ge 3 && $# -le 4 ]] || { backup_error '用法:replay [YYYY-MM-DD HH:MM:SS]'; exit 1; }; replay_binlogs "$2" "$3" "${4:-}" ;; *) printf '用法: %s {prepare|fetch-all|fetch-binlog|start|replay} ...\n' "$0" >&2 exit 1 ;; esac