hfb_sys/scripts/backup-online.sh

#!/usr/bin/env bash
set -euo pipefail
umask 077

# MySQL 在线全量物理备份。XtraBackup 在业务读写期间运行，备份只在
# 上传 payload、manifest、checksum 后最后写入 complete.env，恢复程序只接受
# 带 complete.env 的备份集。

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=backup-common.sh
source "${SCRIPT_DIR}/backup-common.sh"

cleanup_dir=''
cleanup() {
  if [[ -n "${cleanup_dir}" && -d "${cleanup_dir}" ]]; then
    rm -rf -- "${cleanup_dir}"
  fi
}
trap cleanup EXIT

require_backup_config() {
  backup_require_env BACKUP_DIR >/dev/null
  backup_require_env BACKUP_OSS_URI >/dev/null
  backup_require_env BACKUP_PASSPHRASE >/dev/null
  backup_require_env BACKUP_KMS_KEY_ID >/dev/null
  backup_require_env BACKUP_MYSQL_USER >/dev/null
  backup_require_env BACKUP_MYSQL_PASSWORD >/dev/null
}

upload_backup_set() {
  local local_dir="$1"
  local level="$2"
  local ts="$3"
  local server_uuid="$4"
  local oss_uri="$5"
  local remote_dir="${oss_uri}/mysql/full/${level}/${server_uuid}/${ts}"

  backup_log "上传 ${level} 全量备份到 ${remote_dir}"
  backup_oss_upload_kms "${local_dir}/mysql-full.tgz.enc" "${remote_dir}/mysql-full.tgz.enc"
  backup_oss_upload_kms "${local_dir}/manifest.env" "${remote_dir}/manifest.env"
  backup_oss_upload_kms "${local_dir}/mysql-full.tgz.enc.sha256" "${remote_dir}/mysql-full.tgz.enc.sha256"
  # complete.env 必须最后上传，作为远端原子完成标识。
  backup_oss_upload_kms "${local_dir}/complete.env" "${remote_dir}/complete.env"
}

full_backup() {
  backup_require_cmd docker
  backup_require_cmd openssl
  backup_require_cmd ossutil
  require_backup_config

  local backup_dir oss_uri mysql_cid network server_uuid backup_user backup_password ts tmp_dir target_dir
  backup_dir="$(backup_require_env BACKUP_DIR)"
  backup_validate_dir "${backup_dir}"
  oss_uri="$(backup_require_env BACKUP_OSS_URI)"
  backup_validate_oss_uri "${oss_uri}"
  backup_lock_or_exit "${backup_dir}/.full.lock"
  mysql_cid="$(backup_mysql_container_id)"
  network="$(backup_mysql_network "${mysql_cid}")"
  server_uuid="$(backup_server_uuid)"
  backup_user="$(backup_require_env BACKUP_MYSQL_USER)"
  backup_password="$(backup_require_env BACKUP_MYSQL_PASSWORD)"
  ts="$(date +%Y%m%d_%H%M%S)"
  tmp_dir="${backup_dir}/.tmp-full-${ts}-$$"
  target_dir="${backup_dir}/full/daily/${ts}"
  cleanup_dir="${tmp_dir}"
  mkdir -p "${tmp_dir}/xtrabackup"

  cat > "${tmp_dir}/xtrabackup-client.cnf" <<EOF
[client]
user=${backup_user}
password=${backup_password}
host=mysql
EOF
  chmod 600 "${tmp_dir}/xtrabackup-client.cnf"

  backup_log "开始全量热备，实例 ${server_uuid}"
  docker run --rm \
    --network "${network}" \
    --volumes-from "${mysql_cid}:ro" \
    -v "${tmp_dir}/xtrabackup:/backup" \
    -v "${tmp_dir}/xtrabackup-client.cnf:/etc/xtrabackup-client.cnf:ro" \
    "${BACKUP_XTRABACKUP_IMAGE}" \
    xtrabackup \
    --defaults-extra-file=/etc/xtrabackup-client.cnf \
    --backup \
    --target-dir=/backup/data
  rm -f "${tmp_dir}/xtrabackup-client.cnf"
  unset backup_password

  local binlog_file binlog_pos
  binlog_file="$(awk -F$'\t' 'NR == 1 { print $1 }' "${tmp_dir}/xtrabackup/data/xtrabackup_binlog_info")"
  binlog_pos="$(awk -F$'\t' 'NR == 1 { print $2 }' "${tmp_dir}/xtrabackup/data/xtrabackup_binlog_info")"
  if [[ -z "${binlog_file}" || ! "${binlog_pos}" =~ ^[0-9]+$ ]]; then
    backup_error "未读取到有效 xtrabackup_binlog_info"
    exit 1
  fi

  {
    printf 'format_version=1\n'
    printf 'created_at=%s\n' "$(date -Iseconds)"
    printf 'host=%s\n' "$(hostname)"
    printf 'server_uuid=%s\n' "${server_uuid}"
    printf 'mysql_version=%s\n' "$(backup_mysql_root_query 'SELECT VERSION();')"
    printf 'goose_version=%s\n' "$(backup_mysql_root_query 'SELECT COALESCE(MAX(version_id), 0) FROM goose_db_version;')"
    printf 'binlog_file=%s\n' "${binlog_file}"
    printf 'binlog_pos=%s\n' "${binlog_pos}"
    printf 'git_commit=%s\n' "$(git -C "${BACKUP_ROOT_DIR}" rev-parse HEAD 2>/dev/null || true)"
  } > "${tmp_dir}/manifest.env"

  # 关闭包含 XtraBackup 位点的当前 binlog 并归档。全量备份只有在该文件
  # 已有远端 complete 标识后才会被标记完成。
  BACKUP_LOCK_WAIT_SECONDS=120 "${SCRIPT_DIR}/archive-binlog.sh" archive
  if ! find "${backup_dir}/binlog-state/${server_uuid}" -maxdepth 1 -type f \
    -name "${binlog_file}.*.state" -print -quit | grep -q .; then
    backup_error "XtraBackup 位点 ${binlog_file}:${binlog_pos} 尚未完成 binlog 归档"
    exit 1
  fi

  tar czf "${tmp_dir}/mysql-full.tgz" \
    -C "${tmp_dir}/xtrabackup/data" . \
    -C "${tmp_dir}" manifest.env
  backup_encrypt_file "${tmp_dir}/mysql-full.tgz" "${tmp_dir}/mysql-full.tgz.enc"
  rm -f "${tmp_dir}/mysql-full.tgz"

  local payload_sha payload_size
  payload_sha="$(backup_sha256 "${tmp_dir}/mysql-full.tgz.enc")"
  payload_size="$(backup_file_size "${tmp_dir}/mysql-full.tgz.enc")"
  printf '%s  %s\n' "${payload_sha}" 'mysql-full.tgz.enc' > "${tmp_dir}/mysql-full.tgz.enc.sha256"
  backup_write_complete_marker "${tmp_dir}/complete.env" 'mysql-full.tgz.enc' "${payload_sha}" "${payload_size}" "${server_uuid}"

  mkdir -p "${target_dir}"
  mv "${tmp_dir}/mysql-full.tgz.enc" "${target_dir}/"
  mv "${tmp_dir}/mysql-full.tgz.enc.sha256" "${target_dir}/"
  mv "${tmp_dir}/manifest.env" "${target_dir}/"
  mv "${tmp_dir}/complete.env" "${target_dir}/"
  rm -rf "${tmp_dir}/xtrabackup"
  cleanup_dir=''
  rmdir "${tmp_dir}"

  upload_backup_set "${target_dir}" daily "${ts}" "${server_uuid}" "${oss_uri}"
  if [[ "$(date +%u)" == '7' ]]; then
    mkdir -p "${backup_dir}/full/weekly/${ts}"
    ln "${target_dir}/mysql-full.tgz.enc" "${backup_dir}/full/weekly/${ts}/mysql-full.tgz.enc"
    cp "${target_dir}/mysql-full.tgz.enc.sha256" "${target_dir}/manifest.env" "${target_dir}/complete.env" "${backup_dir}/full/weekly/${ts}/"
    upload_backup_set "${target_dir}" weekly "${ts}" "${server_uuid}" "${oss_uri}"
  fi
  if [[ "$(date +%d)" == '01' ]]; then
    mkdir -p "${backup_dir}/full/monthly/${ts}"
    ln "${target_dir}/mysql-full.tgz.enc" "${backup_dir}/full/monthly/${ts}/mysql-full.tgz.enc"
    cp "${target_dir}/mysql-full.tgz.enc.sha256" "${target_dir}/manifest.env" "${target_dir}/complete.env" "${backup_dir}/full/monthly/${ts}/"
    upload_backup_set "${target_dir}" monthly "${ts}" "${server_uuid}" "${oss_uri}"
  fi
  backup_log "全量热备完成：${target_dir}，位点 ${binlog_file}:${binlog_pos}"
}

prune_level() {
  local base="$1"
  local days="$2"
  [[ -d "${base}" ]] || return
  find "${base}" -mindepth 1 -maxdepth 1 -type d -mtime "+${days}" -print0 |
    while IFS= read -r -d '' dir; do
      backup_log "清理过期本地备份：${dir}"
      rm -rf -- "${dir}"
    done
}

prune_backups() {
  local backup_dir daily weekly monthly
  backup_dir="$(backup_require_env BACKUP_DIR)"
  backup_validate_dir "${backup_dir}"
  backup_lock_or_exit "${backup_dir}/.full.lock"
  daily="$(backup_env_value BACKUP_KEEP_DAILY)"; daily="${daily:-14}"
  weekly="$(backup_env_value BACKUP_KEEP_WEEKLY)"; weekly="${weekly:-8}"
  monthly="$(backup_env_value BACKUP_KEEP_MONTHLY)"; monthly="${monthly:-12}"
  [[ "${daily}" =~ ^[0-9]+$ && "${weekly}" =~ ^[0-9]+$ && "${monthly}" =~ ^[0-9]+$ ]] || {
    backup_error 'BACKUP_KEEP_* 必须是非负整数'
    exit 1
  }
  prune_level "${backup_dir}/full/daily" "${daily}"
  prune_level "${backup_dir}/full/weekly" "$((weekly * 7))"
  prune_level "${backup_dir}/full/monthly" "$((monthly * 31))"
}

status() {
  local backup_dir oss_uri
  backup_dir="$(backup_require_env BACKUP_DIR)"
  oss_uri="$(backup_require_env BACKUP_OSS_URI)"
  backup_validate_oss_uri "${oss_uri}"
  backup_log '本地最近完整全量备份：'
  find "${backup_dir}/full" -name complete.env -print 2>/dev/null | sort | tail -8 || true
  backup_log 'OSS 最近完整全量备份：'
  ossutil ls "${oss_uri}/mysql/full/" -r 2>/dev/null | grep 'complete.env' | tail -8 || true
  backup_log 'OSS 最近 binlog 完成标识：'
  ossutil ls "${oss_uri}/mysql/binlog/" -r 2>/dev/null | grep 'complete.env' | tail -8 || true
}

case "${1:-}" in
  full) full_backup ;;
  prune) prune_backups ;;
  status) status ;;
  *)
    printf '用法: %s {full|prune|status}\n' "$0" >&2
    exit 1
    ;;
esac