#!/bin/bash # =========================================== # alert.sh — 服务器异常实时告警（正常时静默，异常才发邮件） # # 检测项： # 磁盘使用 >80% 警告 / >90% 严重 # 内存使用 >80% 警告 / >90% 严重 # 备份时效 >24h 警告 / >48h 严重（含备份缺失） # 服务状态 green/blue/nginx 非 active → 严重 # Docker 容器非 Up → 严重 # SSL 剩余 <30 天警告 / <7 天严重 # 支付卡单 pending 且创建超过 30 分钟 → 警告（列出订单号） # 数据库 不可用 → 严重（支付/会员功能受影响） # # 防刷屏状态机（.alert-state）： # 级别上升 → 立即告警；级别不变 → 超过 ALERT_COOLDOWN_HOURS 再次提醒； # 恢复正常 → 清除状态，不打扰。 # # 部署： # cp scripts/mail-lib.sh scripts/alert.sh → 服务器 /root/rust/scripts/ # cron（root）: # */5 * * * * /root/rust/scripts/alert.sh >> /root/rust/scripts/logs/alert.log 2>&1 # # 用法：/root/rust/scripts/alert.sh [--dry-run] # =========================================== set -u SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) # shellcheck source=mail-lib.sh source "${SCRIPT_DIR}/mail-lib.sh" # ---------- 可配置项（环境变量可覆盖） ---------- : "${ALERT_STATE_FILE:=/root/rust/scripts/.alert-state}" : "${ALERT_COOLDOWN_HOURS:=6}" # 同一级别再次提醒的冷却时间 : "${DB_CONTAINER:=1Panel-postgresql-FtMo}" : "${DB_NAME:=milkydata}" : "${DB_USER:=milkydata}" : "${STUCK_ORDER_MINUTES:=30}" # pending 超过该时长视为卡单 : "${BACKUP_DIR:=/root/rust/backups/milkydata}" # 备份目录 DRY_RUN=false [ "${1:-}" = "--dry-run" ] && DRY_RUN=true # 日志 alert_log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "${MAIL_LOG_DIR}/alert.log" 2>/dev/null || true; } [ "$DRY_RUN" = true ] && alert_log() { echo "[DRY-RUN] $*"; } # ---------- 检测项 ---------- # 每项输出: 级别(0/1/2)|键名|标签|当前值|详情 # 级别: 0=正常 1=警告 2=严重 check_disk() { local pct; pct=$(df -P / | awk 'NR==2{print $5}' | tr -d '%') pct=${pct:-0} local used total; used=$(df -h / | awk 'NR==2{print $3}'); total=$(df -h / | awk 'NR==2{print $2}') local lvl=0 [ "$pct" -ge 90 ] && lvl=2 [ "$pct" -ge 80 ] && [ "$lvl" -eq 0 ] && lvl=1 echo "${lvl}|disk|磁盘使用|${used} / ${total} (${pct}%)|根分区已用 ${pct}%" } check_mem() { local pct; pct=$(free | awk '/Mem/{printf "%.0f", $3/$2 * 100}') pct=${pct:-0} local used total; used=$(free -h | awk '/Mem/{print $3}'); total=$(free -h | awk '/Mem/{print $2}') local lvl=0 [ "$pct" -ge 90 ] && lvl=2 [ "$pct" -ge 80 ] && [ "$lvl" -eq 0 ] && lvl=1 echo "${lvl}|mem|内存使用|${used} / ${total} (${pct}%)|内存已用 ${pct}%" } check_backup() { local dir="$BACKUP_DIR" local last; last=$(ls -1t "$dir" 2>/dev/null | head -1) if [ -z "$last" ]; then echo "2|backup|备份|无备份文件|目录 ${dir} 无任何备份" return fi local age; age=$(( ($(date +%s) - $(stat -c %Y "$dir/$last" 2>/dev/null || echo 0)) / 3600 )) local lvl=0 [ "$age" -ge 48 ] && lvl=2 [ "$age" -ge 24 ] && [ "$lvl" -eq 0 ] && lvl=1 echo "${lvl}|backup|备份时效|${age} 小时前（${last}）|最新备份距今 ${age} 小时" } check_services() { local bad="" for svc in rust-backend-green rust-backend-blue nginx; do local st; st=$(systemctl is-active "$svc" 2>/dev/null || echo "unknown") [ "$st" = "active" ] || bad="${bad}${svc}=${st} " done if [ -n "$bad" ]; then echo "2|services|服务状态|异常服务: ${bad}|systemd 服务未运行" else echo "0|services|服务状态|全部 active|green/blue/nginx 运行正常" fi } check_docker() { local out; out=$(docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null) if [ -z "$out" ]; then if docker info >/dev/null 2>&1; then echo "0|docker|Docker 容器|无运行容器|当前无容器在运行" else echo "2|docker|Docker|不可用|docker 命令执行失败" fi return fi local bad; bad=$(echo "$out" | grep -v " Up " || true) if [ -n "$bad" ]; then echo "2|docker|Docker 容器|$(echo "$out" | grep -c " Up ") 个正常|异常容器: $(echo "$bad" | awk '{print $1}' | tr '\n' ' ')" else echo "0|docker|Docker 容器|$(echo "$out" | grep -c " Up ") 个运行中|全部正常" fi } check_ssl() { local expiry; expiry=$(echo "" | openssl s_client -connect xmclassmate.top:443 -servername xmclassmate.top 2>&1 | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) if [ -z "$expiry" ]; then echo "1|ssl|SSL 证书|检测失败|无法获取证书到期时间" return fi local days; days=$(( ($(date -d "$expiry" +%s 2>/dev/null || echo 0) - $(date +%s)) / 86400 )) local lvl=0 [ "$days" -lt 7 ] && lvl=2 [ "$days" -lt 30 ] && [ "$lvl" -eq 0 ] && lvl=1 echo "${lvl}|ssl|SSL 证书|剩余 ${days} 天（${expiry}）|证书将于 ${expiry} 到期" } # 支付卡单检测（pending 超过 STUCK_ORDER_MINUTES） check_stuck_orders() { local count orders count=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c \ "SELECT count(*) FROM payment_orders WHERE status='pending' AND created_at < NOW() - INTERVAL '${STUCK_ORDER_MINUTES} minutes'" 2>/dev/null) if [ -z "$count" ]; then return # 数据库不可用由 check_db 负责 fi if [ "$count" -gt 0 ]; then orders=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c \ "SELECT order_no || ' (用户' || user_id || ', ' || to_char(created_at, 'MM-DD HH24:MI') || ')' FROM payment_orders WHERE status='pending' AND created_at < NOW() - INTERVAL '${STUCK_ORDER_MINUTES} minutes' ORDER BY created_at" 2>/dev/null | tr '\n' '; ') echo "1|stuck_orders|支付卡单|${count} 笔 pending 超 ${STUCK_ORDER_MINUTES} 分钟|${orders}" else echo "0|stuck_orders|支付卡单|无|无卡单" fi } check_db() { local ok; ok=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c "SELECT 1" 2>/dev/null) if [ -z "$ok" ]; then echo "2|db|数据库|不可用|docker exec psql 失败（容器 ${DB_CONTAINER}）" else echo "0|db|数据库|正常|SELECT 1 成功" fi } # ---------- 状态文件管理 ---------- # 格式: key lvl last_alert_epoch load_state() { [ -f "$ALERT_STATE_FILE" ] && cat "$ALERT_STATE_FILE" || true; } save_state() { local tmp="${ALERT_STATE_FILE}.tmp" { for k in "${!NEW_STATE[@]}"; do echo "$k ${NEW_STATE[$k]}"; done; } > "$tmp" mv "$tmp" "$ALERT_STATE_FILE" 2>/dev/null || true } # 读取状态：$1=key, $2=字段序号（1=级别 2=时间）；状态行格式为 "key lvl time" get_state() { local line; line=$(load_state | awk -v k="$1" '$1==k') [ -n "$line" ] || { echo "-1"; return; } echo "$line" | awk -v f="$2" '{print $(f+1)}' } # ---------- 主流程 ---------- NOW=$(date +%s) COOLDOWN_SEC=$((ALERT_COOLDOWN_HOURS * 3600)) # 收集检测结果（格式: level|key|label|value|detail） RESULTS="" for check in check_disk check_mem check_backup check_services check_docker check_ssl check_db check_stuck_orders; do r=$($check 2>/dev/null) [ -n "$r" ] && RESULTS="${RESULTS}${r}\n" done declare -A NEW_STATE ALERTS="" # 需要发邮件的条目 WARN_COUNT=0 CRIT_COUNT=0 while IFS='|' read -r lvl key label value detail; do [ -z "$key" ] && continue [ "$lvl" -eq 0 ] && continue # 正常项不处理（下面统一清除状态） old_lvl=$(get_state "$key" 1) old_time=$(get_state "$key" 2) [ "$old_time" = "-1" ] && old_time=0 should_alert=false if [ "$lvl" -gt "$old_lvl" ]; then should_alert=true # 级别上升 → 立即告警 elif [ "$lvl" -eq "$old_lvl" ] && [ $((NOW - old_time)) -ge "$COOLDOWN_SEC" ]; then should_alert=true # 同级别冷却期后再次提醒 fi NEW_STATE[$key]="${lvl} ${NOW}" if [ "$should_alert" = true ]; then [ "$lvl" -eq 2 ] && CRIT_COUNT=$((CRIT_COUNT + 1)) [ "$lvl" -eq 1 ] && WARN_COUNT=$((WARN_COUNT + 1)) ALERTS="${ALERTS}$( [ "$lvl" -eq 2 ] && echo "严重" || echo "警告" )${label}${value}${detail}\n" alert_log "告警 [${key}] 级别=${lvl}: ${label} ${value} — ${detail}" fi done < <(printf "%b" "$RESULTS") # 恢复检测：状态文件中存在但本次为 0/未出现的项 → 清除 while read -r key old_lvl _; do [ -z "$key" ] && continue cur_lvl=$(printf "%b" "$RESULTS" | awk -F'|' -v k="$key" '$2==k{print $1; exit}') if [ -z "$cur_lvl" ] || [ "$cur_lvl" -eq 0 ]; then unset 'NEW_STATE[$key]' 2>/dev/null || true alert_log "恢复: ${key} 已恢复正常，状态已清除" fi done < <(load_state) save_state # ---------- 发送告警邮件 ---------- if [ -n "$ALERTS" ]; then if [ "$CRIT_COUNT" -gt 0 ]; then SUBJECT="[ASD 告警] ${CRIT_COUNT} 项严重 / ${WARN_COUNT} 项警告 — $(date '+%m-%d %H:%M')" else SUBJECT="[ASD 告警] ${WARN_COUNT} 项警告 — $(date '+%m-%d %H:%M')" fi BODY=$(cat <

⚠️ ASD 服务器异常告警

$(date '+%Y-%m-%d %H:%M:%S') · $(hostname)
$(printf "%b" "$ALERTS")
级别项目当前值说明
下次同类提醒将在 ${ALERT_COOLDOWN_HOURS} 小时后，恢复正常后不再打扰。
EOF ) if [ "$DRY_RUN" = true ]; then echo "---- 将发送告警邮件 ----"; echo "主题: ${SUBJECT}"; echo "$BODY" | head -30 else if mail_send "$SUBJECT" "$BODY"; then alert_log "告警邮件已发送: ${SUBJECT}" else alert_log "告警邮件发送失败: ${SUBJECT}" fi fi else alert_log "本次检查无异常（$(date '+%H:%M:%S')）" [ "$DRY_RUN" = true ] && echo "当前无异常，无需告警。" fi