258 lines
11 KiB
Bash
258 lines
11 KiB
Bash
#!/bin/bash
|
||
# ===========================================
|
||
# alert.sh — 服务器异常实时告警（正常时静默，异常才发邮件）
|
||
#
|
||
# 检测项：
|
||
# 磁盘使用 >80% 警告 / >90% 严重
|
||
# 内存使用 >80% 警告 / >90% 严重
|
||
# 备份时效 >24h 警告 / >48h 严重（含备份缺失）
|
||
# 服务状态 green/blue/nginx 非 active → 严重
|
||
# Docker 容器非 Up → 严重
|
||
# SSL 剩余 <30 天警告 / <7 天严重
|
||
# 支付卡单 pending 且创建超过 30 分钟 → 警告（列出订单号）
|
||
# 数据库 不可用 → 严重（支付/会员功能受影响）
|
||
#
|
||
# 防刷屏状态机（.alert-state）：
|
||
# 级别上升 → 立即告警；级别不变 → 超过 ALERT_COOLDOWN_HOURS 再次提醒；
|
||
# 恢复正常 → 清除状态，不打扰。
|
||
#
|
||
# 部署：
|
||
# cp scripts/mail-lib.sh scripts/alert.sh → 服务器 /root/rust/scripts/
|
||
# cron（root）:
|
||
# */5 * * * * /root/rust/scripts/alert.sh >> /root/rust/scripts/logs/alert.log 2>&1
|
||
#
|
||
# 用法：/root/rust/scripts/alert.sh [--dry-run]
|
||
# ===========================================
|
||
|
||
set -u
|
||
|
||
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
|
||
# shellcheck source=mail-lib.sh
|
||
source "${SCRIPT_DIR}/mail-lib.sh"
|
||
|
||
# ---------- 可配置项（环境变量可覆盖） ----------
|
||
: "${ALERT_STATE_FILE:=/root/rust/scripts/.alert-state}"
|
||
: "${ALERT_COOLDOWN_HOURS:=6}" # 同一级别再次提醒的冷却时间
|
||
: "${DB_CONTAINER:=1Panel-postgresql-FtMo}"
|
||
: "${DB_NAME:=milkydata}"
|
||
: "${DB_USER:=milkydata}"
|
||
: "${STUCK_ORDER_MINUTES:=30}" # pending 超过该时长视为卡单
|
||
: "${BACKUP_DIR:=/root/rust/backups/milkydata}" # 备份目录
|
||
|
||
DRY_RUN=false
|
||
[ "${1:-}" = "--dry-run" ] && DRY_RUN=true
|
||
|
||
# 日志
|
||
alert_log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "${MAIL_LOG_DIR}/alert.log" 2>/dev/null || true; }
|
||
[ "$DRY_RUN" = true ] && alert_log() { echo "[DRY-RUN] $*"; }
|
||
|
||
# ---------- 检测项 ----------
|
||
# 每项输出: 级别(0/1/2)|键名|标签|当前值|详情
|
||
# 级别: 0=正常 1=警告 2=严重
|
||
|
||
check_disk() {
|
||
local pct; pct=$(df -P / | awk 'NR==2{print $5}' | tr -d '%')
|
||
pct=${pct:-0}
|
||
local used total; used=$(df -h / | awk 'NR==2{print $3}'); total=$(df -h / | awk 'NR==2{print $2}')
|
||
local lvl=0
|
||
[ "$pct" -ge 90 ] && lvl=2
|
||
[ "$pct" -ge 80 ] && [ "$lvl" -eq 0 ] && lvl=1
|
||
echo "${lvl}|disk|磁盘使用|${used} / ${total} (${pct}%)|根分区已用 ${pct}%"
|
||
}
|
||
|
||
check_mem() {
|
||
local pct; pct=$(free | awk '/Mem/{printf "%.0f", $3/$2 * 100}')
|
||
pct=${pct:-0}
|
||
local used total; used=$(free -h | awk '/Mem/{print $3}'); total=$(free -h | awk '/Mem/{print $2}')
|
||
local lvl=0
|
||
[ "$pct" -ge 90 ] && lvl=2
|
||
[ "$pct" -ge 80 ] && [ "$lvl" -eq 0 ] && lvl=1
|
||
echo "${lvl}|mem|内存使用|${used} / ${total} (${pct}%)|内存已用 ${pct}%"
|
||
}
|
||
|
||
check_backup() {
|
||
local dir="$BACKUP_DIR"
|
||
local last; last=$(ls -1t "$dir" 2>/dev/null | head -1)
|
||
if [ -z "$last" ]; then
|
||
echo "2|backup|备份|无备份文件|目录 ${dir} 无任何备份"
|
||
return
|
||
fi
|
||
local age; age=$(( ($(date +%s) - $(stat -c %Y "$dir/$last" 2>/dev/null || echo 0)) / 3600 ))
|
||
local lvl=0
|
||
[ "$age" -ge 48 ] && lvl=2
|
||
[ "$age" -ge 24 ] && [ "$lvl" -eq 0 ] && lvl=1
|
||
echo "${lvl}|backup|备份时效|${age} 小时前（${last}）|最新备份距今 ${age} 小时"
|
||
}
|
||
|
||
check_services() {
|
||
local bad=""
|
||
for svc in rust-backend-green rust-backend-blue nginx; do
|
||
local st; st=$(systemctl is-active "$svc" 2>/dev/null || echo "unknown")
|
||
[ "$st" = "active" ] || bad="${bad}${svc}=${st} "
|
||
done
|
||
if [ -n "$bad" ]; then
|
||
echo "2|services|服务状态|异常服务: ${bad}|systemd 服务未运行"
|
||
else
|
||
echo "0|services|服务状态|全部 active|green/blue/nginx 运行正常"
|
||
fi
|
||
}
|
||
|
||
check_docker() {
|
||
local out; out=$(docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null)
|
||
if [ -z "$out" ]; then
|
||
if docker info >/dev/null 2>&1; then
|
||
echo "0|docker|Docker 容器|无运行容器|当前无容器在运行"
|
||
else
|
||
echo "2|docker|Docker|不可用|docker 命令执行失败"
|
||
fi
|
||
return
|
||
fi
|
||
local bad; bad=$(echo "$out" | grep -v " Up " || true)
|
||
if [ -n "$bad" ]; then
|
||
echo "2|docker|Docker 容器|$(echo "$out" | grep -c " Up ") 个正常|异常容器: $(echo "$bad" | awk '{print $1}' | tr '\n' ' ')"
|
||
else
|
||
echo "0|docker|Docker 容器|$(echo "$out" | grep -c " Up ") 个运行中|全部正常"
|
||
fi
|
||
}
|
||
|
||
check_ssl() {
|
||
local expiry; expiry=$(echo "" | openssl s_client -connect xmclassmate.top:443 -servername xmclassmate.top 2>&1 | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||
if [ -z "$expiry" ]; then
|
||
echo "1|ssl|SSL 证书|检测失败|无法获取证书到期时间"
|
||
return
|
||
fi
|
||
local days; days=$(( ($(date -d "$expiry" +%s 2>/dev/null || echo 0) - $(date +%s)) / 86400 ))
|
||
local lvl=0
|
||
[ "$days" -lt 7 ] && lvl=2
|
||
[ "$days" -lt 30 ] && [ "$lvl" -eq 0 ] && lvl=1
|
||
echo "${lvl}|ssl|SSL 证书|剩余 ${days} 天（${expiry}）|证书将于 ${expiry} 到期"
|
||
}
|
||
|
||
# 支付卡单检测（pending 超过 STUCK_ORDER_MINUTES）
|
||
check_stuck_orders() {
|
||
local count orders
|
||
count=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c \
|
||
"SELECT count(*) FROM payment_orders WHERE status='pending' AND created_at < NOW() - INTERVAL '${STUCK_ORDER_MINUTES} minutes'" 2>/dev/null)
|
||
if [ -z "$count" ]; then
|
||
return # 数据库不可用由 check_db 负责
|
||
fi
|
||
if [ "$count" -gt 0 ]; then
|
||
orders=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c \
|
||
"SELECT order_no || ' (用户' || user_id || ', ' || to_char(created_at, 'MM-DD HH24:MI') || ')' FROM payment_orders WHERE status='pending' AND created_at < NOW() - INTERVAL '${STUCK_ORDER_MINUTES} minutes' ORDER BY created_at" 2>/dev/null | tr '\n' '; ')
|
||
echo "1|stuck_orders|支付卡单|${count} 笔 pending 超 ${STUCK_ORDER_MINUTES} 分钟|${orders}"
|
||
else
|
||
echo "0|stuck_orders|支付卡单|无|无卡单"
|
||
fi
|
||
}
|
||
|
||
check_db() {
|
||
local ok; ok=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c "SELECT 1" 2>/dev/null)
|
||
if [ -z "$ok" ]; then
|
||
echo "2|db|数据库|不可用|docker exec psql 失败（容器 ${DB_CONTAINER}）"
|
||
else
|
||
echo "0|db|数据库|正常|SELECT 1 成功"
|
||
fi
|
||
}
|
||
|
||
# ---------- 状态文件管理 ----------
|
||
# 格式: key lvl last_alert_epoch
|
||
load_state() { [ -f "$ALERT_STATE_FILE" ] && cat "$ALERT_STATE_FILE" || true; }
|
||
|
||
save_state() {
|
||
local tmp="${ALERT_STATE_FILE}.tmp"
|
||
{ for k in "${!NEW_STATE[@]}"; do echo "$k ${NEW_STATE[$k]}"; done; } > "$tmp"
|
||
mv "$tmp" "$ALERT_STATE_FILE" 2>/dev/null || true
|
||
}
|
||
|
||
# 读取状态：$1=key, $2=字段序号（1=级别 2=时间）；状态行格式为 "key lvl time"
|
||
get_state() {
|
||
local line; line=$(load_state | awk -v k="$1" '$1==k')
|
||
[ -n "$line" ] || { echo "-1"; return; }
|
||
echo "$line" | awk -v f="$2" '{print $(f+1)}'
|
||
}
|
||
|
||
# ---------- 主流程 ----------
|
||
NOW=$(date +%s)
|
||
COOLDOWN_SEC=$((ALERT_COOLDOWN_HOURS * 3600))
|
||
|
||
# 收集检测结果（格式: level|key|label|value|detail）
|
||
RESULTS=""
|
||
for check in check_disk check_mem check_backup check_services check_docker check_ssl check_db check_stuck_orders; do
|
||
r=$($check 2>/dev/null)
|
||
[ -n "$r" ] && RESULTS="${RESULTS}${r}\n"
|
||
done
|
||
|
||
declare -A NEW_STATE
|
||
ALERTS="" # 需要发邮件的条目
|
||
WARN_COUNT=0
|
||
CRIT_COUNT=0
|
||
|
||
while IFS='|' read -r lvl key label value detail; do
|
||
[ -z "$key" ] && continue
|
||
[ "$lvl" -eq 0 ] && continue # 正常项不处理（下面统一清除状态）
|
||
|
||
old_lvl=$(get_state "$key" 1)
|
||
old_time=$(get_state "$key" 2)
|
||
[ "$old_time" = "-1" ] && old_time=0
|
||
|
||
should_alert=false
|
||
if [ "$lvl" -gt "$old_lvl" ]; then
|
||
should_alert=true # 级别上升 → 立即告警
|
||
elif [ "$lvl" -eq "$old_lvl" ] && [ $((NOW - old_time)) -ge "$COOLDOWN_SEC" ]; then
|
||
should_alert=true # 同级别冷却期后再次提醒
|
||
fi
|
||
|
||
NEW_STATE[$key]="${lvl} ${NOW}"
|
||
if [ "$should_alert" = true ]; then
|
||
[ "$lvl" -eq 2 ] && CRIT_COUNT=$((CRIT_COUNT + 1))
|
||
[ "$lvl" -eq 1 ] && WARN_COUNT=$((WARN_COUNT + 1))
|
||
ALERTS="${ALERTS}<tr><td style='padding:8px 10px;border-bottom:1px solid #e2e8f0'>$( [ "$lvl" -eq 2 ] && echo "<span style='color:#dc2626'>严重</span>" || echo "<span style='color:#d97706'>警告</span>" )</td><td style='padding:8px 10px;border-bottom:1px solid #e2e8f0'>${label}</td><td style='padding:8px 10px;border-bottom:1px solid #e2e8f0'>${value}</td><td style='padding:8px 10px;border-bottom:1px solid #e2e8f0;color:#475569'>${detail}</td></tr>\n"
|
||
alert_log "告警 [${key}] 级别=${lvl}: ${label} ${value} — ${detail}"
|
||
fi
|
||
done < <(printf "%b" "$RESULTS")
|
||
|
||
# 恢复检测：状态文件中存在但本次为 0/未出现的项 → 清除
|
||
while read -r key old_lvl _; do
|
||
[ -z "$key" ] && continue
|
||
cur_lvl=$(printf "%b" "$RESULTS" | awk -F'|' -v k="$key" '$2==k{print $1; exit}')
|
||
if [ -z "$cur_lvl" ] || [ "$cur_lvl" -eq 0 ]; then
|
||
unset 'NEW_STATE[$key]' 2>/dev/null || true
|
||
alert_log "恢复: ${key} 已恢复正常，状态已清除"
|
||
fi
|
||
done < <(load_state)
|
||
|
||
save_state
|
||
|
||
# ---------- 发送告警邮件 ----------
|
||
if [ -n "$ALERTS" ]; then
|
||
if [ "$CRIT_COUNT" -gt 0 ]; then
|
||
SUBJECT="[ASD 告警] ${CRIT_COUNT} 项严重 / ${WARN_COUNT} 项警告 — $(date '+%m-%d %H:%M')"
|
||
else
|
||
SUBJECT="[ASD 告警] ${WARN_COUNT} 项警告 — $(date '+%m-%d %H:%M')"
|
||
fi
|
||
BODY=$(cat <<EOF
|
||
<html><body style="font-family:-apple-system,'PingFang SC','Microsoft YaHei',sans-serif;max-width:640px;margin:0 auto;padding:16px">
|
||
<h2 style="color:#0f172a;margin:0 0 4px">⚠️ ASD 服务器异常告警</h2>
|
||
<div style="color:#94a3b8;font-size:13px;margin-bottom:16px">$(date '+%Y-%m-%d %H:%M:%S') · $(hostname)</div>
|
||
<table style="border-collapse:collapse;width:100%;font-size:13px">
|
||
<tr><th style="padding:8px 10px;background:#f1f5f9;text-align:left;color:#64748b">级别</th><th style="padding:8px 10px;background:#f1f5f9;text-align:left;color:#64748b">项目</th><th style="padding:8px 10px;background:#f1f5f9;text-align:left;color:#64748b">当前值</th><th style="padding:8px 10px;background:#f1f5f9;text-align:left;color:#64748b">说明</th></tr>
|
||
$(printf "%b" "$ALERTS")
|
||
</table>
|
||
<div style="color:#94a3b8;font-size:12px;margin-top:16px">下次同类提醒将在 ${ALERT_COOLDOWN_HOURS} 小时后，恢复正常后不再打扰。</div>
|
||
</body></html>
|
||
EOF
|
||
)
|
||
if [ "$DRY_RUN" = true ]; then
|
||
echo "---- 将发送告警邮件 ----"; echo "主题: ${SUBJECT}"; echo "$BODY" | head -30
|
||
else
|
||
if mail_send "$SUBJECT" "$BODY"; then
|
||
alert_log "告警邮件已发送: ${SUBJECT}"
|
||
else
|
||
alert_log "告警邮件发送失败: ${SUBJECT}"
|
||
fi
|
||
fi
|
||
else
|
||
alert_log "本次检查无异常（$(date '+%H:%M:%S')）"
|
||
[ "$DRY_RUN" = true ] && echo "当前无异常，无需告警。"
|
||
fi
|