diff --git a/deploy/collect-db-diagnostics.sh b/deploy/collect-db-diagnostics.sh new file mode 100755 index 00000000..b7f01ef3 --- /dev/null +++ b/deploy/collect-db-diagnostics.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# 采集主机、Docker、PostgreSQL、MySQL 诊断信息。 +# 用法: +# bash deploy/collect-db-diagnostics.sh +# bash deploy/collect-db-diagnostics.sh /tmp/diagnostics.txt +# +# 可覆盖的环境变量: +# PG_SERVICE=postgres PG_USER=postgres PG_DB=order_site +# MYSQL_CONTAINER=douyu-login-db MYSQL_USER=root MYSQL_PASSWORD=... + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUTPUT_FILE="${1:-${PROJECT_ROOT}/order-site-diagnostics-$(date +%Y%m%d-%H%M%S).txt}" +PG_SERVICE="${PG_SERVICE:-postgres}" +PG_USER="${PG_USER:-postgres}" +PG_DB="${PG_DB:-order_site}" +MYSQL_CONTAINER="${MYSQL_CONTAINER:-douyu-login-db}" +MYSQL_USER="${MYSQL_USER:-root}" +MYSQL_PASSWORD="${MYSQL_PASSWORD:-}" + +mkdir -p "$(dirname "${OUTPUT_FILE}")" +: >"${OUTPUT_FILE}" +cd "${PROJECT_ROOT}" + +FAILURES=0 + +write_header() { + printf '\n===== %s =====\n' "$1" >>"${OUTPUT_FILE}" +} + +run_cmd() { + local label="$1" + shift + write_header "${label}" + { + printf '$' + printf ' %q' "$@" + printf '\n' + } >>"${OUTPUT_FILE}" + set +e + "$@" >>"${OUTPUT_FILE}" 2>&1 + local status=$? + set -e + printf '[exit=%s]\n' "${status}" >>"${OUTPUT_FILE}" + if [[ ${status} -ne 0 ]]; then + FAILURES=$((FAILURES + 1)) + fi +} + +run_pg_sql() { + local label="$1" + local sql="$2" + write_header "PostgreSQL: ${label}" + printf 'SQL:\n%s\n' "${sql}" >>"${OUTPUT_FILE}" + set +e + docker compose exec -T "${PG_SERVICE}" \ + psql -X -v ON_ERROR_STOP=1 -U "${PG_USER}" -d "${PG_DB}" \ + -P pager=off -P footer=on -c "${sql}" >>"${OUTPUT_FILE}" 2>&1 + local status=$? + set -e + printf '[exit=%s]\n' "${status}" >>"${OUTPUT_FILE}" + if [[ ${status} -ne 0 ]]; then + FAILURES=$((FAILURES + 1)) + fi +} + +run_mysql_sql() { + local label="$1" + local sql="$2" + write_header "MySQL: ${label}" + printf 'SQL:\n%s\n' "${sql}" >>"${OUTPUT_FILE}" + set +e + if [[ -n "${MYSQL_PASSWORD}" ]]; then + docker exec -e "MYSQL_PWD=${MYSQL_PASSWORD}" "${MYSQL_CONTAINER}" \ + mysql --batch --raw --show-warnings -u"${MYSQL_USER}" -e "${sql}" \ + >>"${OUTPUT_FILE}" 2>&1 + else + docker exec "${MYSQL_CONTAINER}" \ + mysql --batch --raw --show-warnings -u"${MYSQL_USER}" -e "${sql}" \ + >>"${OUTPUT_FILE}" 2>&1 + fi + local status=$? + set -e + printf '[exit=%s]\n' "${status}" >>"${OUTPUT_FILE}" + if [[ ${status} -ne 0 ]]; then + FAILURES=$((FAILURES + 1)) + fi +} + +run_cmd "metadata" date -Is +run_cmd "host" uname -a +run_cmd "uptime" uptime +run_cmd "cpu" nproc +run_cmd "memory" free -h +run_cmd "vmstat (5 samples)" vmstat 1 5 +run_cmd "filesystem" df -h +run_cmd "docker ps" docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.RunningFor}}' +run_cmd "docker stats" docker stats --no-stream +run_cmd "docker system df" docker system df +run_cmd "docker disk usage" du -xhd1 /var/lib/docker /var/log + +run_pg_sql "server settings" \ + "SELECT version(), current_database(), current_user; SELECT name, setting, unit FROM pg_settings WHERE name IN ('max_connections','shared_buffers','work_mem','maintenance_work_mem','effective_cache_size','track_io_timing','log_min_duration_statement','autovacuum','autovacuum_max_workers') ORDER BY name;" +run_pg_sql "activity summary" \ + "SELECT COALESCE(state,'') AS state, COALESCE(wait_event_type,'') AS wait_event_type, COALESCE(wait_event,'') AS wait_event, count(*) AS connections FROM pg_stat_activity GROUP BY 1,2,3 ORDER BY connections DESC;" +run_pg_sql "active queries" \ + "SELECT pid, application_name, client_addr, state, wait_event_type, wait_event, clock_timestamp()-query_start AS duration, left(regexp_replace(query, '[[:space:]]+', ' ', 'g'), 1200) AS query FROM pg_stat_activity WHERE state <> 'idle' AND pid <> pg_backend_pid() ORDER BY query_start;" +run_pg_sql "blocking and locks" \ + "SELECT blocked.pid AS blocked_pid, clock_timestamp()-blocked.query_start AS blocked_for, blocking.pid AS blocking_pid, clock_timestamp()-blocking.query_start AS blocking_for, left(regexp_replace(blocked.query, '[[:space:]]+', ' ', 'g'), 500) AS blocked_query, left(regexp_replace(blocking.query, '[[:space:]]+', ' ', 'g'), 500) AS blocking_query FROM pg_stat_activity blocked JOIN LATERAL unnest(pg_blocking_pids(blocked.pid)) AS b(pid) ON true JOIN pg_stat_activity blocking ON blocking.pid=b.pid ORDER BY blocked.query_start;" +run_pg_sql "top pg_stat_statements" \ + "SELECT calls, round(total_exec_time::numeric,2) AS total_ms, round(mean_exec_time::numeric,2) AS mean_ms, rows, left(regexp_replace(query, '[[:space:]]+', ' ', 'g'), 1000) AS query FROM pg_stat_statements ORDER BY total_exec_time DESC LIMIT 30;" +run_pg_sql "largest tables" \ + "SELECT schemaname, relname, pg_size_pretty(pg_total_relation_size(relid)) AS total_size, pg_size_pretty(pg_relation_size(relid)) AS table_size, pg_size_pretty(pg_indexes_size(relid)) AS index_size, n_live_tup, n_dead_tup, last_autovacuum, last_autoanalyze FROM pg_stat_user_tables ORDER BY pg_total_relation_size(relid) DESC LIMIT 30;" +run_pg_sql "dead tuple and vacuum health" \ + "SELECT schemaname, relname, n_live_tup, n_dead_tup, CASE WHEN n_live_tup > 0 THEN round(100.0*n_dead_tup/(n_live_tup+n_dead_tup),2) ELSE 0 END AS dead_pct, last_autovacuum, last_vacuum, last_autoanalyze, last_analyze, autovacuum_count, autoanalyze_count FROM pg_stat_user_tables ORDER BY n_dead_tup DESC LIMIT 30;" +run_pg_sql "worker query indexes" \ + "SELECT tablename, indexname, indexdef FROM pg_indexes WHERE tablename IN ('work_orders','work_order_shares','work_order_events','worker_order_cancel_requests','worker_order_feedbacks') ORDER BY tablename, indexname;" +run_pg_sql "retention table counts" \ + "SELECT c.relname AS table_name, c.reltuples::bigint AS estimated_rows, pg_size_pretty(pg_total_relation_size(c.oid)) AS total_size FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relname IN ('orders','kuaishou_industry_vouchers','task_events','webhook_events','admin_audit_logs','work_order_events') ORDER BY pg_total_relation_size(c.oid) DESC;" + +run_mysql_sql "server status" \ + "SELECT VERSION() AS version, @@hostname AS hostname, @@max_connections AS max_connections, @@innodb_buffer_pool_size AS innodb_buffer_pool_size, @@innodb_flush_log_at_trx_commit AS flush_log_at_trx_commit, @@sync_binlog AS sync_binlog;" +run_mysql_sql "process list" "SHOW FULL PROCESSLIST;" +run_mysql_sql "innodb status" "SHOW ENGINE INNODB STATUS;" +run_mysql_sql "largest tables" \ + "SELECT table_schema, table_name, engine, table_rows, ROUND((data_length+index_length)/1024/1024,2) AS total_mb, ROUND(data_length/1024/1024,2) AS data_mb, ROUND(index_length/1024/1024,2) AS index_mb FROM information_schema.tables WHERE table_schema NOT IN ('information_schema','performance_schema','mysql','sys') ORDER BY data_length+index_length DESC LIMIT 30;" +run_mysql_sql "innodb counters" \ + "SHOW GLOBAL STATUS WHERE Variable_name IN ('Threads_connected','Threads_running','Created_tmp_tables','Created_tmp_disk_tables','Innodb_buffer_pool_reads','Innodb_buffer_pool_read_requests','Innodb_buffer_pool_pages_dirty','Innodb_log_waits','Innodb_row_lock_waits','Innodb_row_lock_time','Innodb_data_reads','Innodb_data_writes','Innodb_os_log_fsyncs');" + +write_header "next step" +cat >>"${OUTPUT_FILE}" <<'EOF' +EXPLAIN (ANALYZE, BUFFERS) requires the real worker ID parameter. The active-query +SQL above is intentionally captured without replaying it, so this collection does +not modify application data. For a slow query, run EXPLAIN only in a maintenance +window and replace $1 with a representative worker ID: + + EXPLAIN (ANALYZE, BUFFERS, VERBOSE) ; +EOF + +printf '诊断采集完成: %s (failed checks: %s)\n' "${OUTPUT_FILE}" "${FAILURES}" +printf '把这个文件完整发回即可。\n'