193 lines
10 KiB
Bash
Executable File
193 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -Eeuo pipefail
|
||
|
||
# 采集主机、Docker、PostgreSQL、MySQL 诊断信息。
|
||
# 用法:
|
||
# bash deploy/collect-db-diagnostics.sh
|
||
# bash deploy/collect-db-diagnostics.sh /tmp/diagnostics.txt
|
||
#
|
||
# 可覆盖的环境变量:
|
||
# PG_SERVICE=postgres PG_USER=postgres PG_DB=order_site
|
||
# ENV_FILE=/root/douyu_login_py/.env
|
||
# MYSQL_CONTAINER=douyu-login-db MYSQL_DIAGNOSTIC_USER=root MYSQL_PASSWORD=...
|
||
#
|
||
# 未显式传入密码时，会从 ENV_FILE 读取 MYSQL_ROOT_PASSWORD（root）或
|
||
# MYSQL_PASSWORD（非 root）。脚本不会把密码写入诊断文件。
|
||
|
||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||
OUTPUT_FILE="${1:-${PROJECT_ROOT}/order-site-diagnostics-$(date +%Y%m%d-%H%M%S).txt}"
|
||
|
||
# 生产环境的 MySQL 通常由独立的 douyu_login_py Compose 项目管理。
|
||
# 自动寻找包含 MySQL 密码配置的 .env，也允许用 ENV_FILE 指定其他路径。
|
||
ENV_FILE="${ENV_FILE:-}"
|
||
if [[ -z "${ENV_FILE}" ]]; then
|
||
for candidate in \
|
||
"${PROJECT_ROOT}/.env" \
|
||
"${PROJECT_ROOT}/../douyu_login_py/.env" \
|
||
"/root/douyu_login_py/.env"; do
|
||
if [[ -f "${candidate}" ]] && grep -qE '^[[:space:]]*MYSQL_(ROOT_)?PASSWORD=' "${candidate}"; then
|
||
ENV_FILE="${candidate}"
|
||
break
|
||
fi
|
||
done
|
||
fi
|
||
|
||
# 显式环境变量优先于 .env，避免服务器临时指定的诊断目标被覆盖。
|
||
EXPLICIT_MYSQL_CONTAINER="${MYSQL_CONTAINER:-}"
|
||
EXPLICIT_MYSQL_USER="${MYSQL_USER:-}"
|
||
EXPLICIT_MYSQL_PASSWORD="${MYSQL_PASSWORD:-}"
|
||
if [[ -n "${ENV_FILE}" && -f "${ENV_FILE}" ]]; then
|
||
set -a
|
||
# shellcheck disable=SC1090
|
||
. "${ENV_FILE}"
|
||
set +a
|
||
fi
|
||
|
||
if [[ -n "${EXPLICIT_MYSQL_CONTAINER}" ]]; then MYSQL_CONTAINER="${EXPLICIT_MYSQL_CONTAINER}"; fi
|
||
if [[ -n "${EXPLICIT_MYSQL_PASSWORD}" ]]; then MYSQL_PASSWORD="${EXPLICIT_MYSQL_PASSWORD}"; fi
|
||
|
||
PG_SERVICE="${PG_SERVICE:-postgres}"
|
||
PG_USER="${PG_USER:-postgres}"
|
||
PG_DB="${PG_DB:-order_site}"
|
||
MYSQL_CONTAINER="${MYSQL_CONTAINER:-douyu-login-db}"
|
||
if [[ -n "${EXPLICIT_MYSQL_USER}" ]]; then
|
||
MYSQL_USER="${EXPLICIT_MYSQL_USER}"
|
||
else
|
||
# douyu_login_py/.env 的 MYSQL_USER 是应用账号；诊断默认需要 root
|
||
# 才能读取 PROCESSLIST、InnoDB 状态和全局性能计数器。
|
||
MYSQL_USER="${MYSQL_DIAGNOSTIC_USER:-root}"
|
||
fi
|
||
if [[ -z "${EXPLICIT_MYSQL_PASSWORD}" && "${MYSQL_USER}" == "root" && -n "${MYSQL_ROOT_PASSWORD:-}" ]]; then
|
||
MYSQL_PASSWORD="${MYSQL_ROOT_PASSWORD}"
|
||
else
|
||
MYSQL_PASSWORD="${MYSQL_PASSWORD:-}"
|
||
fi
|
||
|
||
mkdir -p "$(dirname "${OUTPUT_FILE}")"
|
||
: >"${OUTPUT_FILE}"
|
||
cd "${PROJECT_ROOT}"
|
||
|
||
FAILURES=0
|
||
|
||
write_header() {
|
||
printf '\n===== %s =====\n' "$1" >>"${OUTPUT_FILE}"
|
||
}
|
||
|
||
run_cmd() {
|
||
local label="$1"
|
||
shift
|
||
write_header "${label}"
|
||
{
|
||
printf '$'
|
||
printf ' %q' "$@"
|
||
printf '\n'
|
||
} >>"${OUTPUT_FILE}"
|
||
set +e
|
||
"$@" >>"${OUTPUT_FILE}" 2>&1
|
||
local status=$?
|
||
set -e
|
||
printf '[exit=%s]\n' "${status}" >>"${OUTPUT_FILE}"
|
||
if [[ ${status} -ne 0 ]]; then
|
||
FAILURES=$((FAILURES + 1))
|
||
fi
|
||
}
|
||
|
||
run_pg_sql() {
|
||
local label="$1"
|
||
local sql="$2"
|
||
write_header "PostgreSQL: ${label}"
|
||
printf 'SQL:\n%s\n' "${sql}" >>"${OUTPUT_FILE}"
|
||
set +e
|
||
docker compose exec -T "${PG_SERVICE}" \
|
||
psql -X -v ON_ERROR_STOP=1 -U "${PG_USER}" -d "${PG_DB}" \
|
||
-P pager=off -P footer=on -c "${sql}" >>"${OUTPUT_FILE}" 2>&1
|
||
local status=$?
|
||
set -e
|
||
printf '[exit=%s]\n' "${status}" >>"${OUTPUT_FILE}"
|
||
if [[ ${status} -ne 0 ]]; then
|
||
FAILURES=$((FAILURES + 1))
|
||
fi
|
||
}
|
||
|
||
run_mysql_sql() {
|
||
local label="$1"
|
||
local sql="$2"
|
||
write_header "MySQL: ${label}"
|
||
printf 'SQL:\n%s\n' "${sql}" >>"${OUTPUT_FILE}"
|
||
set +e
|
||
if [[ -n "${MYSQL_PASSWORD}" ]]; then
|
||
docker exec -e "MYSQL_PWD=${MYSQL_PASSWORD}" "${MYSQL_CONTAINER}" \
|
||
mysql --batch --raw --show-warnings -u"${MYSQL_USER}" -e "${sql}" \
|
||
>>"${OUTPUT_FILE}" 2>&1
|
||
else
|
||
docker exec "${MYSQL_CONTAINER}" \
|
||
mysql --batch --raw --show-warnings -u"${MYSQL_USER}" -e "${sql}" \
|
||
>>"${OUTPUT_FILE}" 2>&1
|
||
fi
|
||
local status=$?
|
||
set -e
|
||
printf '[exit=%s]\n' "${status}" >>"${OUTPUT_FILE}"
|
||
if [[ ${status} -ne 0 ]]; then
|
||
FAILURES=$((FAILURES + 1))
|
||
fi
|
||
}
|
||
|
||
write_header "diagnostic configuration"
|
||
printf 'env_file=%s\nmysql_container=%s\nmysql_user=%s\n' \
|
||
"${ENV_FILE:-<none>}" "${MYSQL_CONTAINER}" "${MYSQL_USER}" >>"${OUTPUT_FILE}"
|
||
|
||
run_cmd "metadata" date -Is
|
||
run_cmd "host" uname -a
|
||
run_cmd "uptime" uptime
|
||
run_cmd "cpu" nproc
|
||
run_cmd "memory" free -h
|
||
run_cmd "vmstat (5 samples)" vmstat 1 5
|
||
run_cmd "filesystem" df -h
|
||
run_cmd "docker ps" docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.RunningFor}}'
|
||
run_cmd "docker stats" docker stats --no-stream
|
||
run_cmd "docker system df" docker system df
|
||
run_cmd "docker disk usage" du -xhd1 /var/lib/docker /var/log
|
||
|
||
run_pg_sql "server settings" \
|
||
"SELECT version(), current_database(), current_user; SELECT name, setting, unit FROM pg_settings WHERE name IN ('max_connections','shared_buffers','work_mem','maintenance_work_mem','effective_cache_size','track_io_timing','log_min_duration_statement','autovacuum','autovacuum_max_workers') ORDER BY name;"
|
||
run_pg_sql "activity summary" \
|
||
"SELECT COALESCE(state,'<null>') AS state, COALESCE(wait_event_type,'<none>') AS wait_event_type, COALESCE(wait_event,'<none>') AS wait_event, count(*) AS connections FROM pg_stat_activity GROUP BY 1,2,3 ORDER BY connections DESC;"
|
||
run_pg_sql "active queries" \
|
||
"SELECT pid, application_name, client_addr, state, wait_event_type, wait_event, clock_timestamp()-query_start AS duration, left(regexp_replace(query, '[[:space:]]+', ' ', 'g'), 1200) AS query FROM pg_stat_activity WHERE state <> 'idle' AND pid <> pg_backend_pid() ORDER BY query_start;"
|
||
run_pg_sql "blocking and locks" \
|
||
"SELECT blocked.pid AS blocked_pid, clock_timestamp()-blocked.query_start AS blocked_for, blocking.pid AS blocking_pid, clock_timestamp()-blocking.query_start AS blocking_for, left(regexp_replace(blocked.query, '[[:space:]]+', ' ', 'g'), 500) AS blocked_query, left(regexp_replace(blocking.query, '[[:space:]]+', ' ', 'g'), 500) AS blocking_query FROM pg_stat_activity blocked JOIN LATERAL unnest(pg_blocking_pids(blocked.pid)) AS b(pid) ON true JOIN pg_stat_activity blocking ON blocking.pid=b.pid ORDER BY blocked.query_start;"
|
||
run_pg_sql "top pg_stat_statements" \
|
||
"SELECT calls, round(total_exec_time::numeric,2) AS total_ms, round(mean_exec_time::numeric,2) AS mean_ms, rows, left(regexp_replace(query, '[[:space:]]+', ' ', 'g'), 1000) AS query FROM pg_stat_statements ORDER BY total_exec_time DESC LIMIT 30;"
|
||
run_pg_sql "largest tables" \
|
||
"SELECT schemaname, relname, pg_size_pretty(pg_total_relation_size(relid)) AS total_size, pg_size_pretty(pg_relation_size(relid)) AS table_size, pg_size_pretty(pg_indexes_size(relid)) AS index_size, n_live_tup, n_dead_tup, last_autovacuum, last_autoanalyze FROM pg_stat_user_tables ORDER BY pg_total_relation_size(relid) DESC LIMIT 30;"
|
||
run_pg_sql "dead tuple and vacuum health" \
|
||
"SELECT schemaname, relname, n_live_tup, n_dead_tup, CASE WHEN n_live_tup > 0 THEN round(100.0*n_dead_tup/(n_live_tup+n_dead_tup),2) ELSE 0 END AS dead_pct, last_autovacuum, last_vacuum, last_autoanalyze, last_analyze, autovacuum_count, autoanalyze_count FROM pg_stat_user_tables ORDER BY n_dead_tup DESC LIMIT 30;"
|
||
run_pg_sql "worker query indexes" \
|
||
"SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename IN ('work_orders','work_order_shares','work_order_events','worker_order_cancel_requests','worker_order_feedbacks') ORDER BY tablename, indexname;"
|
||
run_pg_sql "retention table counts" \
|
||
"SELECT c.relname AS table_name, c.reltuples::bigint AS estimated_rows, pg_size_pretty(pg_total_relation_size(c.oid)) AS total_size FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relname IN ('orders','kuaishou_industry_vouchers','task_events','webhook_events','admin_audit_logs','work_order_events') ORDER BY pg_total_relation_size(c.oid) DESC;"
|
||
|
||
run_mysql_sql "server status" \
|
||
"SELECT VERSION() AS version, @@hostname AS hostname, @@max_connections AS max_connections, @@innodb_buffer_pool_size AS innodb_buffer_pool_size, @@innodb_flush_log_at_trx_commit AS flush_log_at_trx_commit, @@sync_binlog AS sync_binlog, @@performance_schema AS performance_schema, @@slow_query_log AS slow_query_log, @@long_query_time AS long_query_time;"
|
||
run_mysql_sql "process list" "SHOW FULL PROCESSLIST;"
|
||
run_mysql_sql "innodb status" "SHOW ENGINE INNODB STATUS;"
|
||
run_mysql_sql "largest tables" \
|
||
"SELECT table_schema, table_name, engine, table_rows, ROUND((data_length+index_length)/1024/1024,2) AS total_mb, ROUND(data_length/1024/1024,2) AS data_mb, ROUND(index_length/1024/1024,2) AS index_mb FROM information_schema.tables WHERE table_schema NOT IN ('information_schema','performance_schema','mysql','sys') ORDER BY data_length+index_length DESC LIMIT 30;"
|
||
run_mysql_sql "innodb counters" \
|
||
"SHOW GLOBAL STATUS WHERE Variable_name IN ('Threads_connected','Threads_running','Max_used_connections','Threads_created','Aborted_connects','Created_tmp_tables','Created_tmp_disk_tables','Com_commit','Com_rollback','Innodb_buffer_pool_reads','Innodb_buffer_pool_read_requests','Innodb_buffer_pool_pages_dirty','Innodb_buffer_pool_wait_free','Innodb_log_waits','Innodb_log_writes','Innodb_os_log_fsyncs','Innodb_os_log_pending_fsyncs','Innodb_row_lock_waits','Innodb_row_lock_current_waits','Innodb_row_lock_time','Innodb_data_reads','Innodb_data_writes');"
|
||
run_mysql_sql "statement digest" \
|
||
"SELECT SCHEMA_NAME, DIGEST_TEXT, COUNT_STAR, ROUND(SUM_TIMER_WAIT/1000000000000,2) AS total_seconds, ROUND(AVG_TIMER_WAIT/1000000000,2) AS mean_ms, SUM_ROWS_EXAMINED, SUM_ROWS_SENT, SUM_NO_INDEX_USED, SUM_NO_GOOD_INDEX_USED FROM performance_schema.events_statements_summary_by_digest WHERE SCHEMA_NAME IS NOT NULL ORDER BY SUM_TIMER_WAIT DESC LIMIT 30;"
|
||
|
||
write_header "next step"
|
||
cat >>"${OUTPUT_FILE}" <<'EOF'
|
||
EXPLAIN (ANALYZE, BUFFERS) requires the real worker ID parameter. The active-query
|
||
SQL above is intentionally captured without replaying it, so this collection does
|
||
not modify application data. For a slow query, run EXPLAIN only in a maintenance
|
||
window and replace $1 with a representative worker ID:
|
||
|
||
EXPLAIN (ANALYZE, BUFFERS, VERBOSE) <captured SQL with $1 replaced by a value>;
|
||
EOF
|
||
|
||
printf '诊断采集完成: %s (failed checks: %s)\n' "${OUTPUT_FILE}" "${FAILURES}"
|
||
printf '把这个文件完整发回即可。\n'
|