diff --git a/backend/.env.prod.example b/backend/.env.prod.example index aace48e..b5db876 100644 --- a/backend/.env.prod.example +++ b/backend/.env.prod.example @@ -48,26 +48,23 @@ LOG_ENABLE_CONSOLE=false LOG_ENABLE_FILE=true LOG_RETAIN_DAYS=14 -# MinIO 容器初始化变量。迁移完成前仍保留,用于回退与校验。 -MINIO_ROOT_USER=change-minio-user -MINIO_ROOT_PASSWORD=change-minio-password -# 当前主对象存储。使用内置 MinIO 时,STORAGE_* 密钥必须和 MINIO_ROOT_* 保持一致。 -STORAGE_ENDPOINT=http://minio:9000 -STORAGE_BUCKET=hfb-sys -STORAGE_ACCESS_KEY_ID=change-minio-user -STORAGE_SECRET_ACCESS_KEY=change-minio-password -STORAGE_REGION= -STORAGE_BUCKET_LOOKUP=auto - -# OSS 迁移镜像端。迁移期间设置后,新上传文件会同时写入 MinIO 与 OSS。 +# 主对象存储:图片等数据已迁移到阿里云 OSS,MinIO 已从生产编排移除。 # 杭州 ECS 应使用内网 Endpoint:https://oss-cn-hangzhou-internal.aliyuncs.com # OSS 的 S3 兼容访问使用 cn-hangzhou 区域和 DNS Bucket 寻址。 +STORAGE_ENDPOINT=https://oss-cn-hangzhou-internal.aliyuncs.com +STORAGE_BUCKET=hfb-sys-assets +STORAGE_ACCESS_KEY_ID=change-oss-access-key-id +STORAGE_SECRET_ACCESS_KEY=change-oss-access-key-secret +STORAGE_REGION=cn-hangzhou +STORAGE_BUCKET_LOOKUP=dns + +# 历史镜像写入配置,迁移完成后保持为空。 STORAGE_MIRROR_ENDPOINT= -STORAGE_MIRROR_BUCKET=hfb-sys-assets +STORAGE_MIRROR_BUCKET= STORAGE_MIRROR_ACCESS_KEY_ID= STORAGE_MIRROR_SECRET_ACCESS_KEY= -STORAGE_MIRROR_REGION=cn-hangzhou -STORAGE_MIRROR_BUCKET_LOOKUP=dns +STORAGE_MIRROR_REGION= +STORAGE_MIRROR_BUCKET_LOOKUP= # 生产环境建议接入真实短信服务;未配置时不要使用 mock 对外运营。 SMS_PROVIDER=aliyun @@ -102,3 +99,25 @@ FIELD_ENCRYPTION_LEGACY_KEY= EXTERNAL_UPLOAD_SECRET= # 可选:逗号分隔的 IP 或 CIDR 白名单,例如 203.0.113.10,10.0.0.0/8。 EXTERNAL_UPLOAD_ALLOWED_IPS= + +# 在线备份(scripts/backup-online.sh / scripts/archive-binlog.sh)。所有项为生产必填。 +# 本地备份根目录,必须是独立数据盘上的 /data 子目录;包含加密全量备份与 binlog 状态。 +BACKUP_DIR=/data/backups +# 隔离恢复演练的数据目录,必须与 BACKUP_DIR 不同;prepare 后的 MySQL 原始数据只放在这里。 +BACKUP_RESTORE_DIR=/data/restore +# 备份专用 OSS Bucket(与业务 Bucket 分离),ossutil 使用。 +# 生成方式:ossutil config 交互配置,或环境变量 OSS_ACCESS_KEY_ID / OSS_ACCESS_KEY_SECRET。 +BACKUP_OSS_URI=oss://hfb-backup +# 客户端加密密码短语(独立保管,与业务密钥一同纳入 KMS/密码管理)。 +# 生成方式:openssl rand -hex 32 +BACKUP_PASSPHRASE=change-to-random-backup-passphrase +# OSS 服务端 KMS 加密 Key ID,备份脚本强制使用,不可留空。 +BACKUP_KMS_KEY_ID=change-kms-key-id +# XtraBackup 专用 MySQL 账号,部署脚本会幂等创建并授予最小备份权限。 +BACKUP_MYSQL_USER=hfb_backup +# 生成方式:openssl rand -hex 24。不要和 MySQL 应用账号或 root 密码复用。 +BACKUP_MYSQL_PASSWORD=change-backup-mysql-password +# 本地保留策略(OSS 侧用 Bucket 生命周期规则管理,见脚本头部注释)。 +BACKUP_KEEP_DAILY=14 +BACKUP_KEEP_WEEKLY=8 +BACKUP_KEEP_MONTHLY=12 diff --git a/deploy/README.md b/deploy/README.md index c97d804..97221c2 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -8,7 +8,7 @@ https://选号网域名 # CADDY_SHOW_DOMAIN(可选),hfb_show 静态 https://monitor.主站域名 # Beszel 监控 ``` -生产部署使用 Caddy 作为公网入口,只暴露宿主机 `80`、`443` 端口。Caddy 会自动申请和续签 HTTPS 证书,并直接托管前端静态资源(主站 `dist` 与可选的选号网 `show-dist` 在构建 Caddy 镜像时打入);后端、MySQL、Redis、MinIO 均通过 Docker 内网通信。选号网与主站**共用同一 backend**,浏览器访问选号网域名下的 `/api/*` 由 Caddy 反代到本机 backend,**不会直连数据库**。 +生产部署使用 Caddy 作为公网入口,只暴露宿主机 `80`、`443` 端口。Caddy 会自动申请和续签 HTTPS 证书,并直接托管前端静态资源(主站 `dist` 与可选的选号网 `show-dist` 在构建 Caddy 镜像时打入);后端、MySQL、Redis 均通过 Docker 内网通信,图片对象存储使用阿里云 OSS。选号网与主站**共用同一 backend**,浏览器访问选号网域名下的 `/api/*` 由 Caddy 反代到本机 backend,**不会直连数据库**。 部署前请确认: @@ -32,12 +32,16 @@ MYSQL_ROOT_PASSWORD MYSQL_PASSWORD MYSQL_DSN JWT_SECRET -MINIO_ROOT_USER -MINIO_ROOT_PASSWORD STORAGE_ACCESS_KEY_ID STORAGE_SECRET_ACCESS_KEY BACKEND_UID BACKEND_GID +BACKUP_DIR +BACKUP_RESTORE_DIR +BACKUP_OSS_URI +BACKUP_PASSPHRASE +BACKUP_KMS_KEY_ID +BACKUP_MYSQL_PASSWORD ``` `BACKEND_UID`、`BACKEND_GID` 填写服务器业务用户的数值 ID,例如: @@ -49,29 +53,68 @@ id -g yml 后端容器会使用该 UID/GID 运行。部署脚本由 root 执行时会自动把已有日志修正为该属主;非 root 执行且发现旧的 root 日志时,会给出需要执行的 `chown` 命令后停止。 -### 迁移对象存储到阿里云 OSS +### 对象存储:阿里云 OSS -创建私有 Bucket 后,先保持 `STORAGE_*` 指向当前 MinIO,在 `backend/.env` 配置 OSS 镜像端: +图片等业务文件存储在私有 OSS Bucket,`backend/.env` 的 `STORAGE_*` 直接指向 OSS: ```text -STORAGE_MIRROR_ENDPOINT=https://oss-cn-hangzhou-internal.aliyuncs.com -STORAGE_MIRROR_BUCKET=hfb-sys-assets -STORAGE_MIRROR_ACCESS_KEY_ID=change-oss-access-key-id -STORAGE_MIRROR_SECRET_ACCESS_KEY=change-oss-access-key-secret -STORAGE_MIRROR_REGION=cn-hangzhou -STORAGE_MIRROR_BUCKET_LOOKUP=dns +STORAGE_ENDPOINT=https://oss-cn-hangzhou-internal.aliyuncs.com +STORAGE_BUCKET=hfb-sys-assets +STORAGE_ACCESS_KEY_ID=change-oss-access-key-id +STORAGE_SECRET_ACCESS_KEY=change-oss-access-key-secret +STORAGE_REGION=cn-hangzhou +STORAGE_BUCKET_LOOKUP=dns ``` -RAM 凭证只授予该 Bucket 的列举、读取、写入对象权限,不要使用阿里云主账号 AccessKey。镜像端启用后,新上传文件会同步写入 MinIO 和 OSS;任一镜像写入失败时上传会失败,避免数据库引用未同步文件。 +RAM 凭证只授予该 Bucket 的列举、读取、写入对象权限,不要使用阿里云主账号 AccessKey。杭州 ECS 必须使用内网 Endpoint,避免跨公网访问。 -部署支持镜像配置的镜像后,执行以下命令同步历史对象;命令可重复执行,不会删除 MinIO 数据: +历史 MinIO 数据已通过 `scripts/migrate-minio-to-oss.sh` 迁移完成,生产编排不再包含 MinIO;旧 MinIO volume 观察期结束后可删除。 + +### 数据库在线备份与时间点恢复 + +生产 MySQL 开启了 `ROW` 格式 binlog 和崩溃安全刷盘。备份期间业务持续读写:每日用 XtraBackup 做物理全量备份,每分钟轮转并归档已关闭的 binlog。备份对象先做客户端加密,再强制以 OSS KMS 加密上传;远端仅在 payload、checksum 和 manifest 都上传成功后写入 `complete.env`。 + +部署脚本会创建或更新 `BACKUP_MYSQL_USER`,该用户只有 XtraBackup 所需的全局备份权限和业务库只读权限,不开放远程 root。 + +备份 Bucket 必须独立于业务 Bucket,备份 RAM 身份仅授予该 Bucket 的列举、读写对象及指定 KMS Key 的加解密权限。推荐 Bucket 开启版本控制和不可变保留策略。 + +部署机需安装 `ossutil`、`openssl` 和 `flock`,并使用与 cron 相同的业务用户完成 `ossutil` 凭证配置;部署脚本会在启动服务前检查这些依赖。 ```bash -./scripts/migrate-minio-to-oss.sh -./scripts/migrate-minio-to-oss.sh --verify +# 每日全量热备,本地留存并上传 OSS 备份桶 +./scripts/backup-online.sh full + +# binlog 分钟级归档,配合全量实现最近保留窗口内的时间点恢复 +./scripts/archive-binlog.sh archive + +# 校验本地状态对应的远端完整对象 / 清理过期本地备份 +./scripts/archive-binlog.sh verify +./scripts/backup-online.sh status +./scripts/backup-online.sh prune ``` -校验通过后,将 `STORAGE_*` 改为同一组 OSS 配置,再清空所有 `STORAGE_MIRROR_*` 配置并重新部署。杭州 ECS 必须使用 `oss-cn-hangzhou-internal.aliyuncs.com`,避免跨公网访问。MinIO 保留至少 30 天后再考虑下线。 +推荐 cron(以部署用户运行,并把日志写到受限目录): + +```cron +* * * * * cd /srv/hfb_sys && ./scripts/archive-binlog.sh archive >> /data/backups/archive-binlog.log 2>&1 +30 4 * * * cd /srv/hfb_sys && ./scripts/backup-online.sh full >> /data/backups/full-backup.log 2>&1 +15 5 * * * cd /srv/hfb_sys && ./scripts/backup-online.sh prune >> /data/backups/prune.log 2>&1 +10 5 * * 0 cd /srv/hfb_sys && ./scripts/archive-binlog.sh verify >> /data/backups/archive-verify.log 2>&1 +``` + +OSS 生命周期建议:`mysql/full/daily/` 30 天、`weekly/` 60 天、`monthly/` 365 天、`mysql/binlog/` 至少 45 天。时间点恢复窗口受最早全量备份和 binlog 保留期共同限制。 + +每周必须在隔离目录做一次恢复演练,绝不向生产容器回放: + +```bash +# 从 status 输出选取一个带 complete.env 的全量备份 URI。 +./scripts/restore-online.sh prepare oss://hfb-backup/mysql/full/daily///complete.env +./scripts/restore-online.sh fetch-all /data/restore// +./scripts/restore-online.sh start /data/restore// +./scripts/restore-online.sh replay /data/restore// hfb-restore- '2026-08-16 20:00:00' +``` + +恢复容器使用 `--network none`,只可通过 `docker exec` 检查。演练结束后手动删除名为 `hfb-restore-*` 的恢复容器和 `/data/restore` 对应目录;生产容器不在恢复工具的可选目标中。 ### 同机部署选号网(hfb_show,可选) @@ -108,12 +151,11 @@ HFB_SHOW_DIR=../hfb_show 脚本会自动完成: -- 检查 `backend/.env` 是否仍指向 `127.0.0.1`、`localhost` 或占位值。 -- 使用内置 MinIO 时,检查 `STORAGE_ACCESS_KEY_ID` / `STORAGE_SECRET_ACCESS_KEY` 是否和 `MINIO_ROOT_USER` / `MINIO_ROOT_PASSWORD` 一致。 +- 检查 `backend/.env` 是否仍指向 `127.0.0.1`、`localhost`、占位值或已下线的 MinIO。 - 构建并启动生产容器。 - 使用配置的非 root UID/GID 运行后端,并检查日志目录属主。 - 通过 Caddy 自动申请或续签 HTTPS 证书。 -- 等待 MySQL、Redis、MinIO 就绪。 +- 等待 MySQL、Redis 就绪。 - 按顺序执行尚未应用的数据库迁移。 - 重启后端并检查 `https://你的域名/api/health`。 diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index 7a224b8..4553324 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -75,30 +75,6 @@ services: timeout: 5s retries: 10 - minio: - image: minio/minio:RELEASE.2025-09-07T16-13-09Z - restart: unless-stopped - command: server /data --console-address ":9001" - env_file: - - ../backend/.env - environment: - TZ: Asia/Shanghai - volumes: - - minio_data:/data - deploy: - resources: - limits: - memory: 512M - cpus: '0.5' - reservations: - memory: 256M - cpus: '0.25' - healthcheck: - test: ["CMD", "mc", "ready", "local"] - interval: 10s - timeout: 5s - retries: 10 - backend: build: context: ../backend @@ -124,8 +100,6 @@ services: condition: service_healthy redis: condition: service_healthy - minio: - condition: service_healthy expose: - "8080" @@ -189,7 +163,6 @@ volumes: caddy_config: mysql_data: redis_data: - minio_data: beszel_data: beszel_socket: beszel_agent_data: diff --git a/deploy/mysql/my.cnf b/deploy/mysql/my.cnf index 4142561..81c389d 100644 --- a/deploy/mysql/my.cnf +++ b/deploy/mysql/my.cnf @@ -8,3 +8,13 @@ default-character-set = utf8mb4 character-set-server = utf8mb4 collation-server = utf8mb4_unicode_ci init-connect = 'SET NAMES utf8mb4' + +# 时间点恢复基础:binlog 由 scripts/archive-binlog.sh 每分钟归档到 OSS 备份桶。 +# 本地按 7 天自动过期,归档与本地保留策略见 scripts/backup-online.sh。 +server-id = 1 +log_bin = mysql-bin +binlog_format = ROW +binlog_row_image = FULL +binlog_expire_logs_seconds = 604800 +sync_binlog = 1 +innodb_flush_log_at_trx_commit = 1 diff --git a/scripts/archive-binlog.sh b/scripts/archive-binlog.sh new file mode 100755 index 0000000..2283ded --- /dev/null +++ b/scripts/archive-binlog.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +# 分钟级 binlog 归档。每次轮转后只归档已关闭文件;每个对象以 +# server_uuid、原始文件名和原始 sha256 命名,避免实例重建后的编号碰撞。 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=backup-common.sh +source "${SCRIPT_DIR}/backup-common.sh" + +MYSQL_DATA_DIR='/var/lib/mysql' +work_dir='' +cleanup() { + if [[ -n "${work_dir}" && -d "${work_dir}" ]]; then + rm -rf -- "${work_dir}" + fi +} +trap cleanup EXIT + +require_archive_config() { + backup_require_env BACKUP_DIR >/dev/null + backup_require_env BACKUP_OSS_URI >/dev/null + backup_require_env BACKUP_PASSPHRASE >/dev/null + backup_require_env BACKUP_KMS_KEY_ID >/dev/null +} + +closed_binlogs() { + local mysql_cid="$1" + docker exec "${mysql_cid}" sh -c \ + 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket -N -B -e "SHOW BINARY LOGS;"' \ + 2>/dev/null | awk 'NR > 1 { print previous } { previous = $1 } END { }' +} + +state_last_index() { + local state_dir="$1" + local value + [[ -f "${state_dir}/last-archived.env" ]] || return + value="$(awk -F= '$1 == "binlog_index" { print $2; exit }' "${state_dir}/last-archived.env")" + [[ "${value}" =~ ^[0-9]+$ ]] && printf '%s' "${value}" +} + +write_state() { + local state_dir="$1" filename="$2" index="$3" raw_sha="$4" cipher_sha="$5" remote_dir="$6" + { + printf 'filename=%s\n' "${filename}" + printf 'binlog_index=%s\n' "${index}" + printf 'raw_sha256=%s\n' "${raw_sha}" + printf 'cipher_sha256=%s\n' "${cipher_sha}" + printf 'remote_dir=%s\n' "${remote_dir}" + } > "${state_dir}/${filename}.${raw_sha}.state" + cp "${state_dir}/${filename}.${raw_sha}.state" "${state_dir}/last-archived.env" +} + +archive_one() { + local mysql_cid="$1" state_dir="$2" server_uuid="$3" oss_uri="$4" filename="$5" + local source_path source_sha raw_file raw_sha cipher_file cipher_sha cipher_size binlog_index object_id remote_dir + source_path="${MYSQL_DATA_DIR}/${filename}" + if ! docker exec "${mysql_cid}" test -f "${source_path}" >/dev/null 2>&1; then + backup_error "待归档 binlog 已从 MySQL 数据目录消失:${filename};PITR 链路已断裂" + exit 1 + fi + raw_file="${work_dir}/${filename}" + docker cp "${mysql_cid}:${source_path}" "${raw_file}" + raw_sha="$(backup_sha256 "${raw_file}")" + source_sha="$(docker exec "${mysql_cid}" sha256sum "${source_path}" | awk '{print $1}')" + [[ "${source_sha}" == "${raw_sha}" ]] || { + backup_error "binlog 拷贝校验失败:${filename}" + exit 1 + } + if compgen -G "${state_dir}/${filename}.${raw_sha}.state" >/dev/null; then + return + fi + binlog_index="$(backup_binlog_index "${filename}")" + object_id="${filename}.${raw_sha}" + remote_dir="${oss_uri}/mysql/binlog/${server_uuid}/${object_id}" + cipher_file="${work_dir}/${object_id}.enc" + backup_encrypt_file "${raw_file}" "${cipher_file}" + cipher_sha="$(backup_sha256 "${cipher_file}")" + cipher_size="$(backup_file_size "${cipher_file}")" + printf '%s %s\n' "${cipher_sha}" 'binlog.enc' > "${work_dir}/${object_id}.enc.sha256" + { + printf 'format_version=1\n' + printf 'server_uuid=%s\n' "${server_uuid}" + printf 'source_binlog=%s\n' "${filename}" + printf 'binlog_index=%s\n' "${binlog_index}" + printf 'raw_sha256=%s\n' "${raw_sha}" + printf 'payload=binlog.enc\n' + printf 'payload_sha256=%s\n' "${cipher_sha}" + printf 'payload_size_bytes=%s\n' "${cipher_size}" + printf 'created_at=%s\n' "$(date -Iseconds)" + } > "${work_dir}/${object_id}.complete.env" + + backup_oss_upload_kms "${cipher_file}" "${remote_dir}/binlog.enc" + backup_oss_upload_kms "${work_dir}/${object_id}.enc.sha256" "${remote_dir}/binlog.enc.sha256" + # complete.env 最后上传,恢复程序只使用有该文件的 binlog。 + backup_oss_upload_kms "${work_dir}/${object_id}.complete.env" "${remote_dir}/complete.env" + write_state "${state_dir}" "${filename}" "${binlog_index}" "${raw_sha}" "${cipher_sha}" "${remote_dir}" + rm -f -- "${raw_file}" "${cipher_file}" "${work_dir}/${object_id}.enc.sha256" "${work_dir}/${object_id}.complete.env" +} + +archive() { + backup_require_cmd docker + backup_require_cmd ossutil + backup_require_cmd openssl + require_archive_config + local backup_dir oss_uri mysql_cid server_uuid state_dir first_binlog last_index first_index expected_index filename index + backup_dir="$(backup_require_env BACKUP_DIR)" + backup_validate_dir "${backup_dir}" + oss_uri="$(backup_require_env BACKUP_OSS_URI)" + backup_validate_oss_uri "${oss_uri}" + backup_lock_or_exit "${backup_dir}/.binlog.lock" + mysql_cid="$(backup_mysql_container_id)" + server_uuid="$(backup_server_uuid)" + state_dir="${backup_dir}/binlog-state/${server_uuid}" + work_dir="${backup_dir}/.tmp-binlog-$$" + mkdir -p "${state_dir}" "${work_dir}" + + docker exec "${mysql_cid}" sh -c \ + 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket -e "FLUSH BINARY LOGS;"' >/dev/null + first_binlog="$(docker exec "${mysql_cid}" sh -c \ + 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket -N -B -e "SHOW BINARY LOGS;"' | head -1 | awk '{print $1}')" + [[ -n "${first_binlog}" ]] || { backup_error '无法读取 binlog 列表'; exit 1; } + last_index="$(state_last_index "${state_dir}" || true)" + first_index="$(backup_binlog_index "${first_binlog}")" + if [[ -n "${last_index}" && "${first_index}" -gt $((last_index + 1)) ]]; then + backup_error "发现未归档 binlog 缺口:已归档至序号 ${last_index},当前最早为 ${first_index}" + exit 1 + fi + + expected_index="${last_index}" + while IFS= read -r filename; do + [[ -n "${filename}" ]] || continue + index="$(backup_binlog_index "${filename}")" + if [[ -n "${expected_index}" && "${index}" -le "${expected_index}" ]]; then + continue + fi + if [[ -n "${expected_index}" && "${index}" -ne $((expected_index + 1)) ]]; then + backup_error "发现未归档 binlog 缺口:期望序号 $((expected_index + 1)),实际为 ${index}" + exit 1 + fi + archive_one "${mysql_cid}" "${state_dir}" "${server_uuid}" "${oss_uri}" "${filename}" + expected_index="${index}" + done < <(closed_binlogs "${mysql_cid}") + backup_log "binlog 归档完成,实例 ${server_uuid}" +} + +verify() { + backup_require_cmd ossutil + require_archive_config + local backup_dir server_uuid state_dir state_file remote_dir + backup_dir="$(backup_require_env BACKUP_DIR)" + backup_validate_dir "${backup_dir}" + backup_lock_or_exit "${backup_dir}/.binlog.lock" + server_uuid="$(backup_server_uuid)" + state_dir="${backup_dir}/binlog-state/${server_uuid}" + [[ -d "${state_dir}" ]] || { backup_error '尚未产生 binlog 归档状态'; exit 1; } + while IFS= read -r -d '' state_file; do + remote_dir="$(awk -F= '$1 == "remote_dir" { print substr($0, index($0, "=") + 1); exit }' "${state_file}")" + [[ -n "${remote_dir}" ]] || { backup_error "归档状态不完整:${state_file}"; exit 1; } + ossutil stat "${remote_dir}/binlog.enc" >/dev/null + ossutil stat "${remote_dir}/binlog.enc.sha256" >/dev/null + ossutil stat "${remote_dir}/complete.env" >/dev/null + done < <(find "${state_dir}" -maxdepth 1 -name '*.state' -print0) + backup_log '全部本地 binlog 归档状态均有远端完整对象' +} + +case "${1:-archive}" in + archive) archive ;; + verify) verify ;; + *) + printf '用法: %s {archive|verify}\n' "$0" >&2 + exit 1 + ;; +esac diff --git a/scripts/backup-common.sh b/scripts/backup-common.sh new file mode 100755 index 0000000..5f47e63 --- /dev/null +++ b/scripts/backup-common.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash + +# 在线备份脚本共用函数。仅供 scripts/backup-online.sh、archive-binlog.sh 和 +# restore-online.sh source,不要单独执行。 + +BACKUP_ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BACKUP_COMPOSE_FILE="${BACKUP_ROOT_DIR}/deploy/docker-compose.prod.yml" +BACKUP_ENV_FILE="${BACKUP_ROOT_DIR}/backend/.env" +BACKUP_XTRABACKUP_IMAGE="${XTRABACKUP_IMAGE:-percona/percona-xtrabackup:8.4}" + +backup_log() { + printf '[backup] %s\n' "$*" >&2 +} + +backup_error() { + printf '[backup] ERROR: %s\n' "$*" >&2 +} + +backup_require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + backup_error "缺少命令:$1" + exit 1 + fi +} + +backup_env_value() { + local key="$1" + awk -F= -v key="${key}" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' "${BACKUP_ENV_FILE}" +} + +backup_require_env() { + local key="$1" + local value + value="$(backup_env_value "${key}")" + if [[ -z "${value}" ]]; then + backup_error "${BACKUP_ENV_FILE} 缺少或未填写 ${key}" + exit 1 + fi + printf '%s' "${value}" +} + +backup_compose() { + docker compose --env-file "${BACKUP_ENV_FILE}" -f "${BACKUP_COMPOSE_FILE}" "$@" +} + +backup_mysql_container_id() { + local cid + cid="$(backup_compose ps -q mysql 2>/dev/null || true)" + if [[ -z "${cid}" ]]; then + backup_error "MySQL 容器未运行" + exit 1 + fi + printf '%s' "${cid}" +} + +backup_mysql_network() { + local network + network="$(docker inspect -f '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$1")" + if [[ -z "${network}" ]]; then + backup_error "无法读取 MySQL Docker 网络" + exit 1 + fi + printf '%s' "${network}" +} + +backup_mysql_root_query() { + backup_compose exec -T mysql sh -c \ + 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket -N -B "$MYSQL_DATABASE" -e "$1"' \ + sh "$1" +} + +backup_validate_dir() { + local dir="$1" + if [[ "${dir}" != /data/* || "${dir}" == "/data" ]]; then + backup_error "备份目录必须是 /data 之下的具体绝对路径,当前值:${dir}" + exit 1 + fi +} + +backup_lock_or_exit() { + local path="$1" + local wait_seconds="${2:-${BACKUP_LOCK_WAIT_SECONDS:-0}}" + backup_require_cmd flock + mkdir -p "$(dirname "${path}")" + exec 9>"${path}" + if [[ ! "${wait_seconds}" =~ ^[0-9]+$ ]]; then + backup_error "锁等待时间必须是非负整数:${wait_seconds}" + exit 1 + fi + if ! flock -w "${wait_seconds}" 9; then + backup_error "已有同类备份任务在执行,拒绝并发运行" + exit 1 + fi +} + +backup_server_uuid() { + local value + value="$(backup_mysql_root_query 'SELECT @@server_uuid;' 2>/dev/null || true)" + if [[ ! "${value}" =~ ^[0-9A-Fa-f-]{36}$ ]]; then + backup_error "无法读取 MySQL server_uuid" + exit 1 + fi + printf '%s' "${value,,}" +} + +backup_binlog_index() { + local filename="$1" + if [[ "${filename}" =~ ^mysql-bin\.([0-9]{6,})$ ]]; then + printf '%d' "$((10#${BASH_REMATCH[1]}))" + return + fi + backup_error "不支持的 binlog 文件名:${filename}" + exit 1 +} + +backup_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + return + fi + shasum -a 256 "$1" | awk '{print $1}' +} + +backup_file_size() { + if [[ "$(uname)" == 'Darwin' ]]; then + stat -f%z "$1" + else + stat -c%s "$1" + fi +} + +backup_encrypt_file() { + local source="$1" + local destination="$2" + local passphrase + passphrase="$(backup_require_env BACKUP_PASSPHRASE)" + BACKUP_PASSPHRASE="${passphrase}" openssl enc -aes-256-cbc -pbkdf2 -iter 600000 -salt \ + -pass env:BACKUP_PASSPHRASE \ + -in "${source}" \ + -out "${destination}" + unset passphrase +} + +backup_decrypt_file() { + local source="$1" + local destination="$2" + local passphrase + passphrase="$(backup_require_env BACKUP_PASSPHRASE)" + BACKUP_PASSPHRASE="${passphrase}" openssl enc -d -aes-256-cbc -pbkdf2 -iter 600000 \ + -pass env:BACKUP_PASSPHRASE \ + -in "${source}" \ + -out "${destination}" + unset passphrase +} + +backup_oss_upload_kms() { + local source="$1" + local destination="$2" + local kms_key_id + kms_key_id="$(backup_require_env BACKUP_KMS_KEY_ID)" + ossutil cp "${source}" "${destination}" \ + --encryption KMS --kms-master-key-id "${kms_key_id}" +} + +backup_validate_oss_uri() { + local uri="$1" + if [[ ! "${uri}" =~ ^oss://[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + backup_error "BACKUP_OSS_URI 必须是 oss://bucket 格式,当前值:${uri}" + exit 1 + fi +} + +backup_oss_download() { + local source="$1" + local destination="$2" + ossutil cp "${source}" "${destination}" +} + +backup_write_complete_marker() { + local path="$1" + local payload_name="$2" + local payload_sha256="$3" + local payload_size="$4" + local server_uuid="$5" + { + printf 'format_version=1\n' + printf 'created_at=%s\n' "$(date -Iseconds)" + printf 'server_uuid=%s\n' "${server_uuid}" + printf 'payload=%s\n' "${payload_name}" + printf 'payload_sha256=%s\n' "${payload_sha256}" + printf 'payload_size_bytes=%s\n' "${payload_size}" + } > "${path}" +} diff --git a/scripts/backup-online.sh b/scripts/backup-online.sh new file mode 100755 index 0000000..bae1cc4 --- /dev/null +++ b/scripts/backup-online.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +# MySQL 在线全量物理备份。XtraBackup 在业务读写期间运行,备份只在 +# 上传 payload、manifest、checksum 后最后写入 complete.env,恢复程序只接受 +# 带 complete.env 的备份集。 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=backup-common.sh +source "${SCRIPT_DIR}/backup-common.sh" + +cleanup_dir='' +cleanup() { + if [[ -n "${cleanup_dir}" && -d "${cleanup_dir}" ]]; then + rm -rf -- "${cleanup_dir}" + fi +} +trap cleanup EXIT + +require_backup_config() { + backup_require_env BACKUP_DIR >/dev/null + backup_require_env BACKUP_OSS_URI >/dev/null + backup_require_env BACKUP_PASSPHRASE >/dev/null + backup_require_env BACKUP_KMS_KEY_ID >/dev/null + backup_require_env BACKUP_MYSQL_USER >/dev/null + backup_require_env BACKUP_MYSQL_PASSWORD >/dev/null +} + +upload_backup_set() { + local local_dir="$1" + local level="$2" + local ts="$3" + local server_uuid="$4" + local oss_uri="$5" + local remote_dir="${oss_uri}/mysql/full/${level}/${server_uuid}/${ts}" + + backup_log "上传 ${level} 全量备份到 ${remote_dir}" + backup_oss_upload_kms "${local_dir}/mysql-full.tgz.enc" "${remote_dir}/mysql-full.tgz.enc" + backup_oss_upload_kms "${local_dir}/manifest.env" "${remote_dir}/manifest.env" + backup_oss_upload_kms "${local_dir}/mysql-full.tgz.enc.sha256" "${remote_dir}/mysql-full.tgz.enc.sha256" + # complete.env 必须最后上传,作为远端原子完成标识。 + backup_oss_upload_kms "${local_dir}/complete.env" "${remote_dir}/complete.env" +} + +full_backup() { + backup_require_cmd docker + backup_require_cmd openssl + backup_require_cmd ossutil + require_backup_config + + local backup_dir oss_uri mysql_cid network server_uuid backup_user backup_password ts tmp_dir target_dir + backup_dir="$(backup_require_env BACKUP_DIR)" + backup_validate_dir "${backup_dir}" + oss_uri="$(backup_require_env BACKUP_OSS_URI)" + backup_validate_oss_uri "${oss_uri}" + backup_lock_or_exit "${backup_dir}/.full.lock" + mysql_cid="$(backup_mysql_container_id)" + network="$(backup_mysql_network "${mysql_cid}")" + server_uuid="$(backup_server_uuid)" + backup_user="$(backup_require_env BACKUP_MYSQL_USER)" + backup_password="$(backup_require_env BACKUP_MYSQL_PASSWORD)" + ts="$(date +%Y%m%d_%H%M%S)" + tmp_dir="${backup_dir}/.tmp-full-${ts}-$$" + target_dir="${backup_dir}/full/daily/${ts}" + cleanup_dir="${tmp_dir}" + mkdir -p "${tmp_dir}/xtrabackup" + + cat > "${tmp_dir}/xtrabackup-client.cnf" </dev/null || true)" + } > "${tmp_dir}/manifest.env" + + # 关闭包含 XtraBackup 位点的当前 binlog 并归档。全量备份只有在该文件 + # 已有远端 complete 标识后才会被标记完成。 + BACKUP_LOCK_WAIT_SECONDS=120 "${SCRIPT_DIR}/archive-binlog.sh" archive + if ! find "${backup_dir}/binlog-state/${server_uuid}" -maxdepth 1 -type f \ + -name "${binlog_file}.*.state" -print -quit | grep -q .; then + backup_error "XtraBackup 位点 ${binlog_file}:${binlog_pos} 尚未完成 binlog 归档" + exit 1 + fi + + tar czf "${tmp_dir}/mysql-full.tgz" \ + -C "${tmp_dir}/xtrabackup/data" . \ + -C "${tmp_dir}" manifest.env + backup_encrypt_file "${tmp_dir}/mysql-full.tgz" "${tmp_dir}/mysql-full.tgz.enc" + rm -f "${tmp_dir}/mysql-full.tgz" + + local payload_sha payload_size + payload_sha="$(backup_sha256 "${tmp_dir}/mysql-full.tgz.enc")" + payload_size="$(backup_file_size "${tmp_dir}/mysql-full.tgz.enc")" + printf '%s %s\n' "${payload_sha}" 'mysql-full.tgz.enc' > "${tmp_dir}/mysql-full.tgz.enc.sha256" + backup_write_complete_marker "${tmp_dir}/complete.env" 'mysql-full.tgz.enc' "${payload_sha}" "${payload_size}" "${server_uuid}" + + mkdir -p "${target_dir}" + mv "${tmp_dir}/mysql-full.tgz.enc" "${target_dir}/" + mv "${tmp_dir}/mysql-full.tgz.enc.sha256" "${target_dir}/" + mv "${tmp_dir}/manifest.env" "${target_dir}/" + mv "${tmp_dir}/complete.env" "${target_dir}/" + rm -rf "${tmp_dir}/xtrabackup" + cleanup_dir='' + rmdir "${tmp_dir}" + + upload_backup_set "${target_dir}" daily "${ts}" "${server_uuid}" "${oss_uri}" + if [[ "$(date +%u)" == '7' ]]; then + mkdir -p "${backup_dir}/full/weekly/${ts}" + ln "${target_dir}/mysql-full.tgz.enc" "${backup_dir}/full/weekly/${ts}/mysql-full.tgz.enc" + cp "${target_dir}/mysql-full.tgz.enc.sha256" "${target_dir}/manifest.env" "${target_dir}/complete.env" "${backup_dir}/full/weekly/${ts}/" + upload_backup_set "${target_dir}" weekly "${ts}" "${server_uuid}" "${oss_uri}" + fi + if [[ "$(date +%d)" == '01' ]]; then + mkdir -p "${backup_dir}/full/monthly/${ts}" + ln "${target_dir}/mysql-full.tgz.enc" "${backup_dir}/full/monthly/${ts}/mysql-full.tgz.enc" + cp "${target_dir}/mysql-full.tgz.enc.sha256" "${target_dir}/manifest.env" "${target_dir}/complete.env" "${backup_dir}/full/monthly/${ts}/" + upload_backup_set "${target_dir}" monthly "${ts}" "${server_uuid}" "${oss_uri}" + fi + backup_log "全量热备完成:${target_dir},位点 ${binlog_file}:${binlog_pos}" +} + +prune_level() { + local base="$1" + local days="$2" + [[ -d "${base}" ]] || return + find "${base}" -mindepth 1 -maxdepth 1 -type d -mtime "+${days}" -print0 | + while IFS= read -r -d '' dir; do + backup_log "清理过期本地备份:${dir}" + rm -rf -- "${dir}" + done +} + +prune_backups() { + local backup_dir daily weekly monthly + backup_dir="$(backup_require_env BACKUP_DIR)" + backup_validate_dir "${backup_dir}" + backup_lock_or_exit "${backup_dir}/.full.lock" + daily="$(backup_env_value BACKUP_KEEP_DAILY)"; daily="${daily:-14}" + weekly="$(backup_env_value BACKUP_KEEP_WEEKLY)"; weekly="${weekly:-8}" + monthly="$(backup_env_value BACKUP_KEEP_MONTHLY)"; monthly="${monthly:-12}" + [[ "${daily}" =~ ^[0-9]+$ && "${weekly}" =~ ^[0-9]+$ && "${monthly}" =~ ^[0-9]+$ ]] || { + backup_error 'BACKUP_KEEP_* 必须是非负整数' + exit 1 + } + prune_level "${backup_dir}/full/daily" "${daily}" + prune_level "${backup_dir}/full/weekly" "$((weekly * 7))" + prune_level "${backup_dir}/full/monthly" "$((monthly * 31))" +} + +status() { + local backup_dir oss_uri + backup_dir="$(backup_require_env BACKUP_DIR)" + oss_uri="$(backup_require_env BACKUP_OSS_URI)" + backup_validate_oss_uri "${oss_uri}" + backup_log '本地最近完整全量备份:' + find "${backup_dir}/full" -name complete.env -print 2>/dev/null | sort | tail -8 || true + backup_log 'OSS 最近完整全量备份:' + ossutil ls "${oss_uri}/mysql/full/" 2>/dev/null | grep 'complete.env' | tail -8 || true + backup_log 'OSS 最近 binlog 完成标识:' + ossutil ls "${oss_uri}/mysql/binlog/" -r 2>/dev/null | grep 'complete.env' | tail -8 || true +} + +case "${1:-}" in + full) full_backup ;; + prune) prune_backups ;; + status) status ;; + *) + printf '用法: %s {full|prune|status}\n' "$0" >&2 + exit 1 + ;; +esac diff --git a/scripts/backup-prod.sh b/scripts/backup-prod.sh deleted file mode 100755 index e58f858..0000000 --- a/scripts/backup-prod.sh +++ /dev/null @@ -1,533 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -COMPOSE_FILE="${ROOT_DIR}/deploy/docker-compose.prod.yml" -BACKEND_ENV="${ROOT_DIR}/backend/.env" -BACKUP_ROOT="${BACKUP_ROOT:-${ROOT_DIR}/backups}" -HELPER_IMAGE="${BACKUP_HELPER_IMAGE:-redis:7.4-alpine}" -if [[ "${BACKUP_ROOT}" != /* ]]; then - BACKUP_ROOT="${ROOT_DIR}/${BACKUP_ROOT}" -fi - -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[1;34m' -NC='\033[0m' - -log() { - printf "${BLUE}[backup]${NC} %s\n" "$*" >&2 -} - -log_success() { - printf "${GREEN}[backup]${NC} %s\n" "$*" >&2 -} - -log_warn() { - printf "${YELLOW}[backup]${NC} %s\n" "$*" >&2 -} - -log_error() { - printf "${RED}[backup]${NC} %s\n" "$*" >&2 -} - -need_cmd() { - if ! command -v "$1" >/dev/null 2>&1; then - log_error "缺少命令:$1" - exit 1 - fi -} - -absolute_dir() { - local dir="$1" - ( - cd "${dir}" - pwd -P - ) -} - -show_help() { - cat << EOF -用法: ./scripts/backup-prod.sh <命令> [选项] - -命令: - backup 创建完整生产备份 - restore <备份目录> 从备份目录恢复生产数据(需要 --yes) - list 列出本机备份目录 - -选项: - --yes 确认执行破坏性恢复,只用于 restore - --no-stop backup 时不主动停止 backend/caddy,适合只读低风险窗口(不推荐) - -h, --help 显示帮助 - -环境变量: - BACKUP_ROOT=/path 备份根目录,默认 ./backups - BACKUP_HELPER_IMAGE=img 用于打包/还原 Docker volume 的镜像,默认 redis:7.4-alpine - -备份内容: - - MySQL 逻辑备份 mysql.sql - - MinIO volume: minio_data.tgz - - Redis volume: redis_data.tgz - - Caddy 证书/配置 volume: caddy_data.tgz, caddy_config.tgz - - backend/.env、deploy 配置、backend/logs - - 当前 Git commit、Docker Compose 状态、goose 迁移状态 - -示例: - ./scripts/backup-prod.sh backup - ./scripts/backup-prod.sh list - ./scripts/backup-prod.sh restore backups/20260618_153000 --yes -EOF -} - -compose() { - local app_env caddy_domain caddy_email admin_base_path - app_env="$(env_value APP_ENV)" - caddy_domain="${CADDY_DOMAIN:-$(env_value CADDY_DOMAIN)}" - caddy_email="${CADDY_EMAIL:-$(env_value CADDY_EMAIL)}" - admin_base_path="${VITE_ADMIN_BASE_PATH:-$(env_value VITE_ADMIN_BASE_PATH)}" - - if [[ "${app_env}" == "production" && (-z "${caddy_domain}" || -z "${caddy_email}") ]]; then - log_error "APP_ENV=production 时必须配置 CADDY_DOMAIN 和 CADDY_EMAIL" - exit 1 - fi - - CADDY_DOMAIN="${caddy_domain:-backup.localhost}" \ - CADDY_EMAIL="${caddy_email:-backup@example.com}" \ - VITE_ADMIN_BASE_PATH="${admin_base_path:-/manage-7x9k2p}" \ - docker compose --env-file "${BACKEND_ENV}" -f "${COMPOSE_FILE}" "$@" -} - -require_env_file() { - if [[ ! -f "${BACKEND_ENV}" ]]; then - log_error "未找到 backend/.env,无法读取生产环境变量" - exit 1 - fi -} - -env_value() { - local key="$1" - awk -F= -v key="${key}" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' "${BACKEND_ENV}" -} - -require_env() { - local key="$1" - local value - value="$(env_value "${key}")" - if [[ -z "${value}" ]]; then - log_error "backend/.env 缺少或未填写 ${key}" - exit 1 - fi - printf "%s" "${value}" -} - -container_id() { - local service="$1" - local cid - cid="$(compose ps -a -q "${service}" 2>/dev/null || true)" - if [[ -z "${cid}" ]]; then - log "创建 ${service} 容器以挂载 volume..." - compose create "${service}" >/dev/null - cid="$(compose ps -a -q "${service}" 2>/dev/null || true)" - fi - if [[ -z "${cid}" ]]; then - log_error "无法找到 ${service} 容器" - exit 1 - fi - printf "%s" "${cid}" -} - -service_was_running() { - local service="$1" - local cid status - cid="$(compose ps -q "${service}" 2>/dev/null || true)" - if [[ -z "${cid}" ]]; then - return 1 - fi - status="$(docker inspect -f '{{.State.Status}}' "${cid}" 2>/dev/null || true)" - [[ "${status}" == "running" ]] -} - -start_services() { - if [[ "$#" -eq 0 ]]; then - return - fi - log "启动服务:$*" - compose up -d "$@" -} - -stop_services() { - if [[ "$#" -eq 0 ]]; then - return - fi - log "停止服务:$*" - compose stop "$@" -} - -start_recorded_services() { - if [[ "$#" -eq 0 ]]; then - return - fi - start_services "$@" -} - -wait_service_ready() { - local service="$1" - local label="$2" - local timeout="${3:-120}" - local attempt cid status - - log "等待 ${label} 就绪..." - for ((attempt = 1; attempt <= timeout; attempt++)); do - cid="$(compose ps -q "${service}" 2>/dev/null || true)" - if [[ -n "${cid}" ]]; then - status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "${cid}" 2>/dev/null || true)" - if [[ "${status}" == "healthy" || "${status}" == "running" ]]; then - log_success "${label} 已就绪" - return - fi - fi - sleep 1 - done - - log_error "${label} 启动超时" - compose logs --tail=80 "${service}" || true - exit 1 -} - -remember_running() { - local service="$1" - local result_array_name="$2" - if service_was_running "${service}"; then - eval "${result_array_name}+=(\"${service}\")" - fi -} - -tar_from_service_volume() { - local service="$1" - local source_dir="$2" - local backup_dir="$3" - local output_name="$4" - local cid - cid="$(container_id "${service}")" - log "备份 ${service}:${source_dir} -> ${output_name}" - docker run --rm \ - --volumes-from "${cid}:ro" \ - -v "${backup_dir}:/backup" \ - "${HELPER_IMAGE}" \ - sh -c "cd '${source_dir}' && tar czf '/backup/${output_name}' ." -} - -restore_service_volume() { - local service="$1" - local target_dir="$2" - local backup_dir="$3" - local input_name="$4" - local cid - if [[ ! -f "${backup_dir}/${input_name}" ]]; then - log_warn "跳过 ${service}:${target_dir},备份文件不存在:${input_name}" - return - fi - cid="$(container_id "${service}")" - log "还原 ${input_name} -> ${service}:${target_dir}" - docker run --rm \ - --volumes-from "${cid}" \ - -v "${backup_dir}:/backup:ro" \ - "${HELPER_IMAGE}" \ - sh -c "rm -rf '${target_dir}'/* '${target_dir}'/.[!.]* '${target_dir}'/..?* 2>/dev/null || true; tar xzf '/backup/${input_name}' -C '${target_dir}'" -} - -copy_if_exists() { - local source="$1" - local target="$2" - if [[ -e "${source}" ]]; then - cp -a "${source}" "${target}" - fi -} - -write_manifest() { - local backup_dir="$1" - { - printf "created_at=%s\n" "$(date -Iseconds)" - printf "host=%s\n" "$(hostname)" - printf "git_commit=%s\n" "$(git -C "${ROOT_DIR}" rev-parse HEAD 2>/dev/null || true)" - printf "git_branch=%s\n" "$(git -C "${ROOT_DIR}" branch --show-current 2>/dev/null || true)" - printf "mysql_database=%s\n" "$(require_env MYSQL_DATABASE)" - printf "storage_bucket=%s\n" "$(require_env STORAGE_BUCKET)" - } > "${backup_dir}/manifest.env" - - git -C "${ROOT_DIR}" log --oneline -5 > "${backup_dir}/git-log.txt" 2>/dev/null || true - compose ps > "${backup_dir}/docker-compose-ps.txt" 2>/dev/null || true - compose config > "${backup_dir}/docker-compose.config.rendered.yml" 2>/dev/null || true -} - -backup_mysql() { - local backup_dir="$1" - log "备份 MySQL -> mysql.sql" - compose exec -T mysql sh -c \ - 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysqldump -uroot --single-transaction --routines --triggers --events --hex-blob --default-character-set=utf8mb4 "$MYSQL_DATABASE"' \ - > "${backup_dir}/mysql.sql" - compose exec -T mysql sh -c 'mysql --version' > "${backup_dir}/mysql-version.txt" 2>/dev/null || true -} - -backup_goose_status() { - local backup_dir="$1" - log "记录 goose 迁移状态" - compose run --rm --no-deps backend /app/goose -dir /app/migrations mysql "$(goose_dsn "$(require_env MYSQL_DSN)")" status \ - > "${backup_dir}/goose-status.txt" 2>&1 || true -} - -goose_dsn() { - local dsn="$1" - if [[ "${dsn}" != *"multiStatements="* ]]; then - if [[ "${dsn}" == *"?"* ]]; then - dsn="${dsn}&multiStatements=true" - else - dsn="${dsn}?multiStatements=true" - fi - fi - printf "%s" "${dsn}" -} - -backup_files() { - local backup_dir="$1" - log "备份配置文件" - mkdir -p "${backup_dir}/config" - copy_if_exists "${BACKEND_ENV}" "${backup_dir}/config/backend.env" - copy_if_exists "${COMPOSE_FILE}" "${backup_dir}/config/docker-compose.prod.yml" - copy_if_exists "${ROOT_DIR}/deploy/caddy/Caddyfile" "${backup_dir}/config/Caddyfile" - copy_if_exists "${ROOT_DIR}/deploy/mysql/my.cnf" "${backup_dir}/config/mysql.my.cnf" - - if [[ -d "${ROOT_DIR}/backend/logs" ]]; then - log "备份后端日志 -> backend_logs.tgz" - tar czf "${backup_dir}/backend_logs.tgz" -C "${ROOT_DIR}/backend" logs - fi -} - -restore_files() { - local backup_dir="$1" - local safety_dir="${BACKUP_ROOT}/pre-restore-files-$(date +%Y%m%d_%H%M%S)" - mkdir -p "${safety_dir}" - - log "恢复配置文件(当前文件会先备份到 ${safety_dir})" - copy_if_exists "${BACKEND_ENV}" "${safety_dir}/backend.env" - copy_if_exists "${COMPOSE_FILE}" "${safety_dir}/docker-compose.prod.yml" - copy_if_exists "${ROOT_DIR}/deploy/caddy/Caddyfile" "${safety_dir}/Caddyfile" - copy_if_exists "${ROOT_DIR}/deploy/mysql/my.cnf" "${safety_dir}/mysql.my.cnf" - - copy_if_exists "${backup_dir}/config/backend.env" "${BACKEND_ENV}" - copy_if_exists "${backup_dir}/config/docker-compose.prod.yml" "${COMPOSE_FILE}" - copy_if_exists "${backup_dir}/config/Caddyfile" "${ROOT_DIR}/deploy/caddy/Caddyfile" - copy_if_exists "${backup_dir}/config/mysql.my.cnf" "${ROOT_DIR}/deploy/mysql/my.cnf" - - if [[ -f "${backup_dir}/backend_logs.tgz" ]]; then - log "恢复后端日志" - mkdir -p "${ROOT_DIR}/backend" - rm -rf "${ROOT_DIR}/backend/logs" - tar xzf "${backup_dir}/backend_logs.tgz" -C "${ROOT_DIR}/backend" - fi -} - -prepare_restore_env() { - local backup_dir="$1" - if [[ ! -f "${BACKEND_ENV}" ]]; then - if [[ ! -f "${backup_dir}/config/backend.env" ]]; then - log_error "当前缺少 backend/.env,备份中也没有 config/backend.env,无法恢复" - exit 1 - fi - log_warn "当前缺少 backend/.env,先从备份恢复环境变量文件" - mkdir -p "$(dirname "${BACKEND_ENV}")" - cp -a "${backup_dir}/config/backend.env" "${BACKEND_ENV}" - fi - if [[ ! -f "${COMPOSE_FILE}" && -f "${backup_dir}/config/docker-compose.prod.yml" ]]; then - log_warn "当前缺少 deploy/docker-compose.prod.yml,先从备份恢复 Compose 文件" - mkdir -p "$(dirname "${COMPOSE_FILE}")" - cp -a "${backup_dir}/config/docker-compose.prod.yml" "${COMPOSE_FILE}" - fi - require_env_file -} - -restore_mysql() { - local backup_dir="$1" - local database - database="$(require_env MYSQL_DATABASE)" - if [[ ! "${database}" =~ ^[A-Za-z0-9_]+$ ]]; then - log_error "MYSQL_DATABASE 只能包含字母、数字和下划线,当前值:${database}" - exit 1 - fi - if [[ ! -f "${backup_dir}/mysql.sql" ]]; then - log_error "缺少 MySQL 备份文件:${backup_dir}/mysql.sql" - exit 1 - fi - - start_services mysql - wait_service_ready mysql "MySQL" - log "重建数据库 ${database}" - compose exec -T mysql sh -c \ - 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot -e "DROP DATABASE IF EXISTS \`$MYSQL_DATABASE\`; CREATE DATABASE \`$MYSQL_DATABASE\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"' - log "导入 MySQL 备份" - compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot "$MYSQL_DATABASE"' < "${backup_dir}/mysql.sql" -} - -backup_all() { - require_env_file - need_cmd docker - local no_stop="$1" - local backup_dir="${BACKUP_ROOT}/$(date +%Y%m%d_%H%M%S)" - local restart_services=() - local backup_ok=0 - - backup_failure_cleanup() { - local exit_code=$? - trap - ERR - if [[ "${no_stop}" == "0" && "${backup_ok}" != "1" ]]; then - log_warn "备份失败,尝试恢复备份前运行的服务" - start_recorded_services "${restart_services[@]+"${restart_services[@]}"}" || true - fi - exit "${exit_code}" - } - trap backup_failure_cleanup ERR - - umask 077 - mkdir -p "${backup_dir}" - chmod 700 "${backup_dir}" - log "备份目录:${backup_dir}" - - if [[ "${no_stop}" == "0" ]]; then - remember_running caddy restart_services - remember_running backend restart_services - stop_services caddy backend - else - log_warn "已开启 --no-stop,备份期间仍可能有写入,建议只在低风险窗口使用" - fi - - start_services mysql - wait_service_ready mysql "MySQL" - backup_mysql "${backup_dir}" - backup_goose_status "${backup_dir}" - - if [[ "${no_stop}" == "0" ]]; then - remember_running minio restart_services - remember_running redis restart_services - stop_services minio redis caddy - fi - - tar_from_service_volume minio /data "${backup_dir}" minio_data.tgz - tar_from_service_volume redis /data "${backup_dir}" redis_data.tgz - tar_from_service_volume caddy /data "${backup_dir}" caddy_data.tgz - tar_from_service_volume caddy /config "${backup_dir}" caddy_config.tgz - backup_files "${backup_dir}" - write_manifest "${backup_dir}" - - if [[ "${no_stop}" == "0" ]]; then - start_recorded_services "${restart_services[@]+"${restart_services[@]}"}" - fi - - backup_ok=1 - trap - ERR - log_success "备份完成:${backup_dir}" - log_warn "备份包含生产密钥,请离线加密保存,不要提交到 Git。" -} - -restore_all() { - need_cmd docker - local backup_dir="$1" - local yes="$2" - local restore_ok=0 - - if [[ -z "${backup_dir}" || ! -d "${backup_dir}" ]]; then - log_error "备份目录不存在:${backup_dir}" - exit 1 - fi - if [[ "${yes}" != "1" ]]; then - log_error "restore 会覆盖当前生产数据库、MinIO、Redis、Caddy volume 和配置文件。确认执行请追加 --yes" - exit 1 - fi - backup_dir="$(absolute_dir "${backup_dir}")" - - restore_failure_cleanup() { - local exit_code=$? - trap - ERR - if [[ "${restore_ok}" != "1" ]]; then - log_warn "恢复失败,尝试启动当前配置下的服务,便于继续排查" - compose up -d || true - fi - exit "${exit_code}" - } - trap restore_failure_cleanup ERR - - prepare_restore_env "${backup_dir}" - log_warn "即将从 ${backup_dir} 恢复,当前生产数据会被覆盖。" - stop_services backend caddy minio redis - restore_files "${backup_dir}" - restore_mysql "${backup_dir}" - - restore_service_volume minio /data "${backup_dir}" minio_data.tgz - restore_service_volume redis /data "${backup_dir}" redis_data.tgz - restore_service_volume caddy /data "${backup_dir}" caddy_data.tgz - restore_service_volume caddy /config "${backup_dir}" caddy_config.tgz - - log "启动生产服务" - compose up -d - restore_ok=1 - trap - ERR - log_success "恢复完成。建议立刻执行健康检查并抽查图片/二维码/订单数据。" -} - -list_backups() { - if [[ ! -d "${BACKUP_ROOT}" ]]; then - log_warn "备份目录不存在:${BACKUP_ROOT}" - return - fi - find "${BACKUP_ROOT}" -mindepth 2 -maxdepth 2 -type f -name manifest.env -exec dirname {} \; | sort -} - -main() { - local command="${1:-}" - local no_stop=0 - local yes=0 - local restore_dir="" - - if [[ -z "${command}" || "${command}" == "-h" || "${command}" == "--help" ]]; then - show_help - exit 0 - fi - shift || true - - case "${command}" in - backup) - while [[ $# -gt 0 ]]; do - case "$1" in - --no-stop) no_stop=1; shift ;; - -h|--help) show_help; exit 0 ;; - *) log_error "未知选项:$1"; show_help; exit 1 ;; - esac - done - backup_all "${no_stop}" - ;; - restore) - restore_dir="${1:-}" - if [[ $# -gt 0 ]]; then - shift - fi - while [[ $# -gt 0 ]]; do - case "$1" in - --yes) yes=1; shift ;; - -h|--help) show_help; exit 0 ;; - *) log_error "未知选项:$1"; show_help; exit 1 ;; - esac - done - restore_all "${restore_dir}" "${yes}" - ;; - list) - list_backups - ;; - *) - log_error "未知命令:${command}" - show_help - exit 1 - ;; - esac -} - -main "$@" diff --git a/scripts/deploy-prod.sh b/scripts/deploy-prod.sh index 8fee5fa..cec8155 100755 --- a/scripts/deploy-prod.sh +++ b/scripts/deploy-prod.sh @@ -146,23 +146,28 @@ validate_env() { exit 1 fi - local app_env caddy_domain caddy_email mysql_dsn redis_addr storage_endpoint backend_uid backend_gid - local minio_root_user minio_root_password storage_access_key_id storage_secret_access_key + local app_env caddy_domain caddy_email mysql_dsn redis_addr storage_endpoint backend_uid backend_gid backup_dir backup_restore_dir backup_oss_uri backup_user app_env="$(require_env APP_ENV)" caddy_domain="$(require_env CADDY_DOMAIN)" caddy_email="$(require_env CADDY_EMAIL)" mysql_dsn="$(require_env MYSQL_DSN)" redis_addr="$(require_env REDIS_ADDR)" storage_endpoint="$(require_env STORAGE_ENDPOINT)" - require_env MYSQL_DATABASE >/dev/null + local mysql_database + mysql_database="$(require_env MYSQL_DATABASE)" require_env MYSQL_ROOT_PASSWORD >/dev/null require_env MYSQL_USER >/dev/null require_env MYSQL_PASSWORD >/dev/null require_env JWT_SECRET >/dev/null - minio_root_user="$(require_env MINIO_ROOT_USER)" - minio_root_password="$(require_env MINIO_ROOT_PASSWORD)" - storage_access_key_id="$(require_env STORAGE_ACCESS_KEY_ID)" - storage_secret_access_key="$(require_env STORAGE_SECRET_ACCESS_KEY)" + require_env STORAGE_ACCESS_KEY_ID >/dev/null + require_env STORAGE_SECRET_ACCESS_KEY >/dev/null + backup_dir="$(require_env BACKUP_DIR)" + backup_restore_dir="$(require_env BACKUP_RESTORE_DIR)" + backup_oss_uri="$(require_env BACKUP_OSS_URI)" + require_env BACKUP_PASSPHRASE >/dev/null + require_env BACKUP_KMS_KEY_ID >/dev/null + backup_user="$(require_env BACKUP_MYSQL_USER)" + require_env BACKUP_MYSQL_PASSWORD >/dev/null backend_uid="$(require_env BACKEND_UID)" backend_gid="$(require_env BACKEND_GID)" @@ -197,14 +202,36 @@ validate_env() { exit 1 fi if [[ "${storage_endpoint}" == *"127.0.0.1"* || "${storage_endpoint}" == *"localhost"* ]]; then - log_error "STORAGE_ENDPOINT 仍指向本机,请改为 http://minio:9000" + log_error "STORAGE_ENDPOINT 仍指向本机,请改为阿里云 OSS Endpoint(如 https://oss-cn-hangzhou.aliyuncs.com)" exit 1 fi - if [[ "${storage_endpoint}" == "http://minio:9000" || "${storage_endpoint}" == "minio:9000" ]]; then - if [[ "${storage_access_key_id}" != "${minio_root_user}" || "${storage_secret_access_key}" != "${minio_root_password}" ]]; then - log_error "使用内置 MinIO 时,STORAGE_ACCESS_KEY_ID/SECRET_ACCESS_KEY 必须和 MINIO_ROOT_USER/ROOT_PASSWORD 保持一致" - exit 1 - fi + if [[ "${storage_endpoint}" == *"minio"* ]]; then + log_error "STORAGE_ENDPOINT 仍指向已下线的 MinIO,请改为阿里云 OSS Endpoint" + exit 1 + fi + if [[ ! "${storage_endpoint}" =~ ^https://oss-[A-Za-z0-9-]+\.aliyuncs\.com/?$ ]]; then + log_error "STORAGE_ENDPOINT 必须是阿里云 OSS HTTPS Endpoint,当前值:${storage_endpoint}" + exit 1 + fi + if [[ "${backup_dir}" != /data/* || "${backup_dir}" == "/data" ]]; then + log_error "BACKUP_DIR 必须是 /data 之下的具体绝对路径,当前值:${backup_dir}" + exit 1 + fi + if [[ "${backup_restore_dir}" != /data/* || "${backup_restore_dir}" == "/data" || "${backup_restore_dir}" == "${backup_dir}" ]]; then + log_error "BACKUP_RESTORE_DIR 必须是与 BACKUP_DIR 不同的 /data 子目录,当前值:${backup_restore_dir}" + exit 1 + fi + if [[ ! "${backup_oss_uri}" =~ ^oss://[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + log_error "BACKUP_OSS_URI 必须是 oss://bucket 格式,当前值:${backup_oss_uri}" + exit 1 + fi + if [[ ! "${backup_user}" =~ ^[A-Za-z0-9_]{1,32}$ ]]; then + log_error "BACKUP_MYSQL_USER 只能包含字母、数字和下划线,长度不超过 32" + exit 1 + fi + if [[ ! "${mysql_database}" =~ ^[A-Za-z0-9_]+$ ]]; then + log_error "MYSQL_DATABASE 只能包含字母、数字和下划线,当前值:${mysql_database}" + exit 1 fi if awk -F= '/^[A-Z0-9_]+=/{ if ($0 ~ /=(change-|保持你的|你的)/) found=1 } END { exit found ? 0 : 1 }' "${BACKEND_ENV}"; then log_error "backend/.env 里还有占位值,请先替换后再部署" @@ -407,6 +434,34 @@ mysql_root_exec() { compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot "$@"' sh "$@" } +sql_string_literal() { + local value="$1" + value="${value//\\/\\\\}" + value="${value//\'/\'\'}" + printf '%s' "${value}" +} + +ensure_mysql_backup_user() { + local database backup_user backup_password escaped_password + database="$(require_env MYSQL_DATABASE)" + backup_user="$(require_env BACKUP_MYSQL_USER)" + backup_password="$(require_env BACKUP_MYSQL_PASSWORD)" + escaped_password="$(sql_string_literal "${backup_password}")" + unset backup_password + + log "配置 MySQL 在线备份账号 ${backup_user}" + printf "CREATE USER IF NOT EXISTS \`%s\`@'%%' IDENTIFIED BY '%s';\n" "${backup_user}" "${escaped_password}" | + compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket' + printf "ALTER USER \`%s\`@'%%' IDENTIFIED BY '%s';\n" "${backup_user}" "${escaped_password}" | + compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket' + printf "GRANT BACKUP_ADMIN, PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO \`%s\`@'%%';\n" "${backup_user}" | + compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket' + printf "GRANT SELECT ON performance_schema.log_status TO \`%s\`@'%%';\n" "${backup_user}" | + compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket' + printf "GRANT SELECT ON \`%s\`.* TO \`%s\`@'%%';\n" "${database}" "${backup_user}" | + compose exec -T mysql sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket' +} + goose_dsn() { local dsn="$1" if [[ "${dsn}" != *"multiStatements="* ]]; then @@ -492,6 +547,9 @@ main() { cd "${ROOT_DIR}" need_cmd docker need_cmd curl + need_cmd openssl + need_cmd ossutil + need_cmd flock validate_env export_caddy_env prepare_log_dir @@ -534,8 +592,8 @@ main() { wait_service_healthy mysql "MySQL" wait_service_healthy redis "Redis" - wait_service_healthy minio "MinIO" wait_service_healthy caddy "Caddy" + ensure_mysql_backup_user reset_database run_migrations diff --git a/scripts/restore-online.sh b/scripts/restore-online.sh new file mode 100755 index 0000000..56d74cc --- /dev/null +++ b/scripts/restore-online.sh @@ -0,0 +1,248 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +# 只用于隔离恢复演练或灾难恢复准备。它从不触碰生产 mysql 容器:prepare +# 在 BACKUP_RESTORE_DIR 下生成数据目录,start 仅启动 --network none 的临时容器。 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=backup-common.sh +source "${SCRIPT_DIR}/backup-common.sh" + +field_value() { + local file="$1" key="$2" + awk -F= -v key="${key}" '$1 == key { print substr($0, index($0, "=") + 1); exit }' "${file}" +} + +require_restore_dir() { + local dir + dir="$(backup_require_env BACKUP_RESTORE_DIR)" + backup_validate_dir "${dir}" + printf '%s' "${dir}" +} + +validate_sha256() { + [[ "$1" =~ ^[0-9a-f]{64}$ ]] || { backup_error "非法 SHA-256:$1"; exit 1; } +} + +validate_server_uuid() { + [[ "$1" =~ ^[0-9a-f-]{36}$ ]] || { backup_error "非法 server_uuid:$1"; exit 1; } +} + +verify_downloaded_payload() { + local payload="$1" checksum_file="$2" complete_file="$3" + local expected actual expected_size actual_size + expected="$(field_value "${complete_file}" payload_sha256)" + expected_size="$(field_value "${complete_file}" payload_size_bytes)" + validate_sha256 "${expected}" + [[ "${expected_size}" =~ ^[0-9]+$ ]] || { backup_error '完成标识中的文件大小不合法'; exit 1; } + actual="$(backup_sha256 "${payload}")" + actual_size="$(backup_file_size "${payload}")" + [[ "${expected}" == "${actual}" && "${expected_size}" == "${actual_size}" ]] || { + backup_error "下载文件校验失败:${payload}" + exit 1 + } + grep -Fqx "${expected} $(basename "${payload}")" "${checksum_file}" || { + backup_error "校验文件内容不匹配:${checksum_file}" + exit 1 + } +} + +prepare_full() { + backup_require_cmd docker + backup_require_cmd ossutil + backup_require_cmd openssl + local complete_uri="$1" + [[ "${complete_uri}" == oss://*/complete.env ]] || { + backup_error 'prepare 参数必须是完整全量备份的 complete.env OSS URI' + exit 1 + } + local restore_dir remote_dir ts server_uuid work_dir download_dir + restore_dir="$(require_restore_dir)" + remote_dir="${complete_uri%/complete.env}" + ts="${remote_dir##*/}" + server_uuid="$(basename "$(dirname "${remote_dir}")")" + validate_server_uuid "${server_uuid}" + work_dir="${restore_dir}/${server_uuid}/${ts}" + download_dir="${work_dir}/.download" + [[ ! -e "${work_dir}" ]] || { backup_error "恢复目录已存在:${work_dir}"; exit 1; } + mkdir -p "${download_dir}" "${work_dir}/data" + trap 'rm -rf -- "${download_dir}"' EXIT + + backup_oss_download "${complete_uri}" "${download_dir}/complete.env" + [[ "$(field_value "${download_dir}/complete.env" payload)" == 'mysql-full.tgz.enc' ]] || { + backup_error '不是受支持的全量备份完成标识' + exit 1 + } + [[ "$(field_value "${download_dir}/complete.env" server_uuid)" == "${server_uuid}" ]] || { + backup_error '备份路径与完成标识的 server_uuid 不一致' + exit 1 + } + backup_oss_download "${remote_dir}/mysql-full.tgz.enc" "${download_dir}/mysql-full.tgz.enc" + backup_oss_download "${remote_dir}/mysql-full.tgz.enc.sha256" "${download_dir}/mysql-full.tgz.enc.sha256" + verify_downloaded_payload "${download_dir}/mysql-full.tgz.enc" "${download_dir}/mysql-full.tgz.enc.sha256" "${download_dir}/complete.env" + backup_decrypt_file "${download_dir}/mysql-full.tgz.enc" "${download_dir}/mysql-full.tgz" + tar xzf "${download_dir}/mysql-full.tgz" -C "${work_dir}/data" + local manifest_uuid binlog_file binlog_pos + manifest_uuid="$(field_value "${work_dir}/data/manifest.env" server_uuid)" + binlog_file="$(field_value "${work_dir}/data/manifest.env" binlog_file)" + binlog_pos="$(field_value "${work_dir}/data/manifest.env" binlog_pos)" + [[ "${manifest_uuid}" == "${server_uuid}" ]] || { backup_error '归档内 manifest 与完成标识不一致'; exit 1; } + backup_binlog_index "${binlog_file}" >/dev/null + [[ "${binlog_pos}" =~ ^[0-9]+$ ]] || { backup_error 'manifest 中 binlog 位点非法'; exit 1; } + docker run --rm -v "${work_dir}/data:/restore" "${BACKUP_XTRABACKUP_IMAGE}" \ + xtrabackup --prepare --target-dir=/restore + { + printf 'server_uuid=%s\n' "${server_uuid}" + printf 'binlog_file=%s\n' "${binlog_file}" + printf 'binlog_pos=%s\n' "${binlog_pos}" + printf 'source_complete_uri=%s\n' "${complete_uri}" + printf 'prepared_at=%s\n' "$(date -Iseconds)" + } > "${work_dir}/prepared.env" + rm -rf -- "${download_dir}" + trap - EXIT + backup_log "全量备份已 prepare:${work_dir}" + backup_log "下一步执行:$0 fetch-all ${work_dir}" +} + +fetch_binlog() { + backup_require_cmd docker + backup_require_cmd ossutil + backup_require_cmd openssl + local prepared_dir="$1" complete_uri="$2" + [[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录:${prepared_dir}"; exit 1; } + [[ "${complete_uri}" == oss://*/complete.env ]] || { backup_error 'binlog 参数必须是 complete.env OSS URI'; exit 1; } + local remote_dir temp_dir complete source_file raw_sha payload server_uuid expected_uuid target_base + remote_dir="${complete_uri%/complete.env}" + temp_dir="${prepared_dir}/.binlog-download-$$" + mkdir -p "${temp_dir}" "${prepared_dir}/binlogs" + trap 'rm -rf -- "${temp_dir}"' EXIT + backup_oss_download "${complete_uri}" "${temp_dir}/complete.env" + source_file="$(field_value "${temp_dir}/complete.env" source_binlog)" + raw_sha="$(field_value "${temp_dir}/complete.env" raw_sha256)" + payload="$(field_value "${temp_dir}/complete.env" payload)" + server_uuid="$(field_value "${temp_dir}/complete.env" server_uuid)" + expected_uuid="$(field_value "${prepared_dir}/prepared.env" server_uuid)" + backup_binlog_index "${source_file}" >/dev/null + validate_sha256 "${raw_sha}" + validate_server_uuid "${server_uuid}" + [[ "${payload}" == 'binlog.enc' && "${server_uuid}" == "${expected_uuid}" ]] || { + backup_error 'binlog 完成标识与恢复目标不匹配' + exit 1 + } + target_base="${prepared_dir}/binlogs/${source_file}.${raw_sha}" + if [[ -f "${target_base}.meta.env" && -f "${target_base}" ]]; then + rm -rf -- "${temp_dir}" + trap - EXIT + return + fi + backup_oss_download "${remote_dir}/binlog.enc" "${temp_dir}/binlog.enc" + backup_oss_download "${remote_dir}/binlog.enc.sha256" "${temp_dir}/binlog.enc.sha256" + verify_downloaded_payload "${temp_dir}/binlog.enc" "${temp_dir}/binlog.enc.sha256" "${temp_dir}/complete.env" + backup_decrypt_file "${temp_dir}/binlog.enc" "${target_base}" + docker run --rm -v "${prepared_dir}/binlogs:/binlogs:ro" mysql:8.4 \ + mysqlbinlog --verify-binlog-checksum "/binlogs/$(basename "${target_base}")" >/dev/null + mv "${temp_dir}/complete.env" "${target_base}.meta.env" + rm -rf -- "${temp_dir}" + trap - EXIT +} + +fetch_all() { + backup_require_cmd ossutil + local prepared_dir="$1" server_uuid oss_uri complete_uri + [[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录:${prepared_dir}"; exit 1; } + server_uuid="$(field_value "${prepared_dir}/prepared.env" server_uuid)" + oss_uri="$(backup_require_env BACKUP_OSS_URI)" + backup_validate_oss_uri "${oss_uri}" + while IFS= read -r complete_uri; do + [[ -n "${complete_uri}" ]] || continue + fetch_binlog "${prepared_dir}" "${complete_uri}" + done < <(ossutil ls "${oss_uri}/mysql/binlog/${server_uuid}/" -r | awk '$NF ~ /\/complete\.env$/ { print $NF }') + backup_log "已下载实例 ${server_uuid} 的全部完整 binlog" +} + +start_isolated_mysql() { + backup_require_cmd docker + local prepared_dir="$1" name="${2:-}" + [[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录:${prepared_dir}"; exit 1; } + if [[ -z "${name}" ]]; then + name="hfb-restore-$(basename "${prepared_dir}")" + fi + [[ "${name}" =~ ^hfb-restore-[A-Za-z0-9_-]+$ ]] || { backup_error '恢复容器名必须以 hfb-restore- 开头'; exit 1; } + [[ -z "$(docker ps -aq -f "name=^/${name}$")" ]] || { backup_error "恢复容器已存在:${name}"; exit 1; } + docker run -d --name "${name}" --network none \ + -e "MYSQL_ROOT_PASSWORD=$(backup_require_env MYSQL_ROOT_PASSWORD)" \ + -v "${prepared_dir}/data:/var/lib/mysql" mysql:8.4 \ + --skip-log-bin --skip-networking=1 >/dev/null + local attempt + for ((attempt = 1; attempt <= 60; attempt++)); do + if docker exec "${name}" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysqladmin -uroot --protocol=socket ping' >/dev/null 2>&1; then + backup_log "隔离 MySQL 已启动:${name}" + return + fi + sleep 1 + done + backup_error "隔离 MySQL 启动失败,请执行 docker logs ${name}" + exit 1 +} + +replay_binlogs() { + backup_require_cmd docker + local prepared_dir="$1" target_container="$2" stop_datetime="${3:-}" + [[ -f "${prepared_dir}/prepared.env" ]] || { backup_error "未 prepare 的恢复目录:${prepared_dir}"; exit 1; } + local production_cid + production_cid="$(backup_mysql_container_id)" + [[ "${target_container}" != "${production_cid}" ]] || { backup_error '禁止向生产 MySQL 回放 binlog'; exit 1; } + docker inspect "${target_container}" >/dev/null + [[ -d "${prepared_dir}/binlogs" ]] || { backup_error '尚未下载 binlog'; exit 1; } + local start_file start_pos metadata index source previous_index found=0 + start_file="$(field_value "${prepared_dir}/prepared.env" binlog_file)" + start_pos="$(field_value "${prepared_dir}/prepared.env" binlog_pos)" + mapfile -t metadata < <( + for metadata in "${prepared_dir}"/binlogs/*.meta.env; do + [[ -f "${metadata}" ]] || continue + index="$(field_value "${metadata}" binlog_index)" + source="$(field_value "${metadata}" source_binlog)" + printf '%010d\t%s\t%s\n' "${index}" "${source}" "${metadata}" + done | sort -n + ) + [[ "${#metadata[@]}" -gt 0 ]] || { backup_error '没有可回放的 binlog'; exit 1; } + local binlog_paths=() + previous_index='' + local row path + for row in "${metadata[@]}"; do + IFS=$'\t' read -r index source metadata <<< "${row}" + if [[ "${source}" == "${start_file}" ]]; then + found=1 + fi + [[ "${found}" == '1' ]] || continue + if [[ -n "${previous_index}" && "${index}" -ne $((previous_index + 1)) ]]; then + backup_error "恢复 binlog 不连续:${previous_index} -> ${index}" + exit 1 + fi + path="/binlogs/$(basename "${metadata%.meta.env}")" + binlog_paths+=("${path}") + previous_index="${index}" + done + [[ "${found}" == '1' ]] || { backup_error "缺少全量备份起始 binlog:${start_file}"; exit 1; } + local mysqlbinlog_args=(mysqlbinlog "--start-position=${start_pos}") + if [[ -n "${stop_datetime}" ]]; then + mysqlbinlog_args+=("--stop-datetime=${stop_datetime}") + fi + mysqlbinlog_args+=("${binlog_paths[@]}") + docker run --rm -i -v "${prepared_dir}/binlogs:/binlogs:ro" mysql:8.4 "${mysqlbinlog_args[@]}" | + docker exec -i "${target_container}" sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --protocol=socket' + backup_log "binlog 已回放至隔离 MySQL:${target_container}" +} + +case "${1:-}" in + prepare) [[ $# -eq 2 ]] || { backup_error '用法:prepare '; exit 1; }; prepare_full "$2" ;; + fetch-all) [[ $# -eq 2 ]] || { backup_error '用法:fetch-all '; exit 1; }; fetch_all "$2" ;; + fetch-binlog) [[ $# -eq 3 ]] || { backup_error '用法:fetch-binlog '; exit 1; }; fetch_binlog "$2" "$3" ;; + start) [[ $# -le 3 && $# -ge 2 ]] || { backup_error '用法:start [container-name]'; exit 1; }; start_isolated_mysql "$2" "${3:-}" ;; + replay) [[ $# -ge 3 && $# -le 4 ]] || { backup_error '用法:replay [YYYY-MM-DD HH:MM:SS]'; exit 1; }; replay_binlogs "$2" "$3" "${4:-}" ;; + *) + printf '用法: %s {prepare|fetch-all|fetch-binlog|start|replay} ...\n' "$0" >&2 + exit 1 + ;; +esac