新增备份监控与定时配置

2026-08-16 23:53:50 +08:00
parent 66cb03ebce
commit f67ee12d72
19 changed files with 612 additions and 3 deletions
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+# 由宿主机 cron 每分钟调用。调度配置来自后台 system_configs，实际执行仍在宿主机，
+# Web 后端不接触 Docker socket、备份载荷或客户端加密密码。
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=backup-common.sh
+source "${SCRIPT_DIR}/backup-common.sh"
+
+config_value() {
+  local key="$1" fallback="$2" value
+  value="$(backup_mysql_root_query "SELECT value FROM system_configs WHERE \`key\` = '${key}' LIMIT 1;" 2>/dev/null || true)"
+  value="${value//$'\n'/}"
+  printf '%s' "${value:-${fallback}}"
+}
+
+write_job_state() {
+  local status_dir="$1" type="$2" status="$3" message="$4"
+  mkdir -p "${status_dir}"
+  {
+    printf 'job_type=%q\n' "${type}"
+    printf 'job_status=%q\n' "${status}"
+    printf 'job_at=%q\n' "$(date -Iseconds)"
+    printf 'job_message=%q\n' "${message}"
+  } > "${status_dir}/last-job.env"
+  chmod 600 "${status_dir}/last-job.env"
+}
+
+run_job() {
+  local status_dir="$1" type="$2" script="$3"
+  shift 3
+  if "${script}" "$@"; then
+    write_job_state "${status_dir}" "${type}" success '执行成功'
+    "${SCRIPT_DIR}/backup-status.sh" write || true
+    return
+  fi
+  local code=$?
+  write_job_state "${status_dir}" "${type}" failed "执行失败（退出码 ${code}）"
+  "${SCRIPT_DIR}/backup-status.sh" write || true
+  return "${code}"
+}
+
+main() {
+  backup_require_cmd docker
+  backup_require_cmd flock
+  local backup_dir status_dir enabled hour minute interval now_hour now_minute
+  backup_dir="$(backup_require_env BACKUP_DIR)"
+  status_dir="${BACKUP_STATUS_DIR:-${backup_dir}/status}"
+  backup_validate_dir "${status_dir}"
+  backup_lock_or_exit "${backup_dir}/.scheduler.lock"
+
+  enabled="$(config_value 'backup.schedule_enabled' 'true')"
+  hour="$(config_value 'backup.full_hour' '4')"
+  minute="$(config_value 'backup.full_minute' '30')"
+  interval="$(config_value 'backup.binlog_interval_minutes' '1')"
+  [[ "${enabled}" == true || "${enabled}" == false ]] || { backup_error 'backup.schedule_enabled 必须为 true 或 false'; exit 1; }
+  [[ "${hour}" =~ ^[0-9]+$ && "${hour}" -le 23 && "${minute}" =~ ^[0-9]+$ && "${minute}" -le 59 ]] || {
+    backup_error '备份全量时间配置无效'
+    exit 1
+  }
+  [[ "${interval}" =~ ^[0-9]+$ && "${interval}" -ge 1 && "${interval}" -le 5 ]] || {
+    backup_error 'binlog 归档间隔必须为 1-5 分钟'
+    exit 1
+  }
+  if [[ "${enabled}" == false ]]; then
+    "${SCRIPT_DIR}/backup-status.sh" write || true
+    exit 0
+  fi
+
+  now_hour="$((10#$(date +%H)))"
+  now_minute="$((10#$(date +%M)))"
+  if ((now_minute % interval == 0)); then
+    run_job "${status_dir}" binlog "${SCRIPT_DIR}/archive-binlog.sh" archive
+  fi
+  if ((now_hour == hour && now_minute == minute)); then
+    run_job "${status_dir}" full "${SCRIPT_DIR}/backup-online.sh" full
+  fi
+  "${SCRIPT_DIR}/backup-status.sh" write || true
+}
+
+main "$@"
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+# 生成供后台只读展示的备份状态摘要。内容不包含密钥、备份载荷或下载地址。
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=backup-common.sh
+source "${SCRIPT_DIR}/backup-common.sh"
+
+json_escape() {
+  local value="$1"
+  value="${value//\\/\\\\}"
+  value="${value//\"/\\\"}"
+  value="${value//$'\n'/ }"
+  value="${value//$'\r'/ }"
+  printf '%s' "${value}"
+}
+
+config_value() {
+  local key="$1" fallback="$2" value
+  value="$(backup_mysql_root_query "SELECT value FROM system_configs WHERE \`key\` = '${key}' LIMIT 1;" 2>/dev/null || true)"
+  value="${value//$'\n'/}"
+  printf '%s' "${value:-${fallback}}"
+}
+
+write_status() {
+  backup_require_cmd docker
+  backup_require_cmd ossutil
+
+  local backup_dir oss_uri status_dir status_file temp_file enabled hour minute interval
+  local local_complete remote_complete binlog_state binlog_file binlog_remote job_type job_status job_at job_message health
+  backup_dir="$(backup_require_env BACKUP_DIR)"
+  oss_uri="$(backup_require_env BACKUP_OSS_URI)"
+  status_dir="${BACKUP_STATUS_DIR:-${backup_dir}/status}"
+  backup_validate_dir "${status_dir}"
+  mkdir -p "${status_dir}"
+  chmod 700 "${status_dir}"
+  status_file="${status_dir}/status.json"
+  temp_file="${status_dir}/.status-$$.json"
+
+  enabled="$(config_value 'backup.schedule_enabled' 'true')"
+  hour="$(config_value 'backup.full_hour' '4')"
+  minute="$(config_value 'backup.full_minute' '30')"
+  interval="$(config_value 'backup.binlog_interval_minutes' '1')"
+  local_complete="$(find "${backup_dir}/full/daily" -name complete.env -type f -print 2>/dev/null | sort | tail -1 || true)"
+  remote_complete="$(ossutil ls "${oss_uri}/mysql/full/daily/" -r 2>/dev/null | awk '$NF ~ /\/complete\.env$/ { latest = $NF } END { print latest }')"
+  binlog_state="$(find "${backup_dir}/binlog-state" -name last-archived.env -type f -print 2>/dev/null | sort | tail -1 || true)"
+  binlog_file=''
+  binlog_remote=''
+  if [[ -n "${binlog_state}" ]]; then
+    binlog_file="$(awk -F= '$1 == "filename" { print substr($0, index($0, "=") + 1); exit }' "${binlog_state}")"
+    binlog_remote="$(awk -F= '$1 == "remote_dir" { print substr($0, index($0, "=") + 1); exit }' "${binlog_state}")"
+  fi
+
+  job_type=''
+  job_status=''
+  job_at=''
+  job_message=''
+  if [[ -f "${status_dir}/last-job.env" ]]; then
+    # shellcheck disable=SC1090
+    source "${status_dir}/last-job.env"
+  fi
+  if [[ -n "${local_complete}" && -n "${remote_complete}" && -n "${binlog_file}" ]]; then
+    health='healthy'
+  else
+    health='warning'
+  fi
+
+  cat > "${temp_file}" <<EOF
+{
+  "generated_at": "$(date -Iseconds)",
+  "health": "$(json_escape "${health}")",
+  "schedule": {
+    "enabled": "$(json_escape "${enabled}")",
+    "full_hour": "$(json_escape "${hour}")",
+    "full_minute": "$(json_escape "${minute}")",
+    "binlog_interval_minutes": "$(json_escape "${interval}")"
+  },
+  "last_full": {
+    "local_complete": "$(json_escape "${local_complete}")",
+    "remote_complete": "$(json_escape "${remote_complete}")"
+  },
+  "last_binlog": {
+    "file": "$(json_escape "${binlog_file}")",
+    "remote_dir": "$(json_escape "${binlog_remote}")"
+  },
+  "last_job": {
+    "type": "$(json_escape "${job_type}")",
+    "status": "$(json_escape "${job_status}")",
+    "at": "$(json_escape "${job_at}")",
+    "message": "$(json_escape "${job_message}")"
+  }
+}
+EOF
+  chmod 640 "${temp_file}"
+  mv "${temp_file}" "${status_file}"
+}
+
+case "${1:-write}" in
+  write) write_status ;;
+  *)
+    printf '用法: %s {write}\n' "$0" >&2
+    exit 1
+    ;;
+esac
@@ -146,7 +146,7 @@ validate_env() {
    exit 1
  fi

-  local app_env caddy_domain caddy_email mysql_dsn redis_addr storage_endpoint backend_uid backend_gid backup_dir backup_restore_dir backup_oss_uri backup_user
+  local app_env caddy_domain caddy_email mysql_dsn redis_addr storage_endpoint backend_uid backend_gid backup_dir backup_status_dir backup_restore_dir backup_oss_uri backup_user
  app_env="$(require_env APP_ENV)"
  caddy_domain="$(require_env CADDY_DOMAIN)"
  caddy_email="$(require_env CADDY_EMAIL)"
@@ -162,6 +162,7 @@ validate_env() {
  require_env STORAGE_ACCESS_KEY_ID >/dev/null
  require_env STORAGE_SECRET_ACCESS_KEY >/dev/null
  backup_dir="$(require_env BACKUP_DIR)"
+  backup_status_dir="$(require_env BACKUP_STATUS_DIR)"
  backup_restore_dir="$(require_env BACKUP_RESTORE_DIR)"
  backup_oss_uri="$(require_env BACKUP_OSS_URI)"
  require_env BACKUP_PASSPHRASE >/dev/null
@@ -216,6 +217,10 @@ validate_env() {
    log_error "BACKUP_DIR 必须是 /data 之下的具体绝对路径，当前值：${backup_dir}"
    exit 1
  fi
+  if [[ "${backup_status_dir}" != "${backup_dir}/"* ]]; then
+    log_error "BACKUP_STATUS_DIR 必须位于 BACKUP_DIR 之下，当前值：${backup_status_dir}"
+    exit 1
+  fi
  if [[ "${backup_restore_dir}" != /data/* || "${backup_restore_dir}" == "/data" || "${backup_restore_dir}" == "${backup_dir}" ]]; then
    log_error "BACKUP_RESTORE_DIR 必须是与 BACKUP_DIR 不同的 /data 子目录，当前值：${backup_restore_dir}"
    exit 1
@@ -403,6 +408,24 @@ prepare_log_dir() {
  chmod 700 "${BACKEND_LOG_DIR}"
 }

+prepare_backup_status_dir() {
+  local backend_uid backend_gid status_dir owner_mismatch
+  backend_uid="$(require_env BACKEND_UID)"
+  backend_gid="$(require_env BACKEND_GID)"
+  status_dir="$(require_env BACKUP_STATUS_DIR)"
+  mkdir -p "${status_dir}"
+  owner_mismatch="$(find "${status_dir}" \( ! -uid "${backend_uid}" -o ! -gid "${backend_gid}" \) -print -quit)"
+  if [[ -n "${owner_mismatch}" ]]; then
+    if [[ "${EUID}" -ne 0 ]]; then
+      log_error "备份状态目录存在属主不匹配的文件：${owner_mismatch}"
+      log_error "请先执行：sudo chown -R ${backend_uid}:${backend_gid} ${status_dir}"
+      exit 1
+    fi
+    chown -R "${backend_uid}:${backend_gid}" "${status_dir}"
+  fi
+  chmod 700 "${status_dir}"
+}
+
 wait_service_healthy() {
  local service="$1"
  local label="$2"
@@ -557,6 +580,7 @@ main() {
  validate_env
  export_caddy_env
  prepare_log_dir
+  prepare_backup_status_dir
  # 选号网需在 build caddy 之前准备 show-dist 与 conf.d
  prepare_show_site