feat: 邮件告警与每日报告优化（HTML+UTF-8、阈值实时告警、支付卡单检测、重试与状态机）
Some checks failed
Deploy Backend / deploy (push) Has been cancelled
Some checks failed
Deploy Backend / deploy (push) Has been cancelled
This commit is contained in:
257
scripts/alert.sh
Normal file
257
scripts/alert.sh
Normal file
@@ -0,0 +1,257 @@
|
||||
#!/bin/bash
|
||||
# ===========================================
|
||||
# alert.sh — 服务器异常实时告警（正常时静默，异常才发邮件）
|
||||
#
|
||||
# 检测项：
|
||||
# 磁盘使用 >80% 警告 / >90% 严重
|
||||
# 内存使用 >80% 警告 / >90% 严重
|
||||
# 备份时效 >24h 警告 / >48h 严重（含备份缺失）
|
||||
# 服务状态 green/blue/nginx 非 active → 严重
|
||||
# Docker 容器非 Up → 严重
|
||||
# SSL 剩余 <30 天警告 / <7 天严重
|
||||
# 支付卡单 pending 且创建超过 30 分钟 → 警告（列出订单号）
|
||||
# 数据库 不可用 → 严重（支付/会员功能受影响）
|
||||
#
|
||||
# 防刷屏状态机（.alert-state）：
|
||||
# 级别上升 → 立即告警；级别不变 → 超过 ALERT_COOLDOWN_HOURS 再次提醒；
|
||||
# 恢复正常 → 清除状态，不打扰。
|
||||
#
|
||||
# 部署：
|
||||
# cp scripts/mail-lib.sh scripts/alert.sh → 服务器 /root/rust/scripts/
|
||||
# cron（root）:
|
||||
# */5 * * * * /root/rust/scripts/alert.sh >> /root/rust/scripts/logs/alert.log 2>&1
|
||||
#
|
||||
# 用法：/root/rust/scripts/alert.sh [--dry-run]
|
||||
# ===========================================
|
||||
|
||||
set -u
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=mail-lib.sh
|
||||
source "${SCRIPT_DIR}/mail-lib.sh"
|
||||
|
||||
# ---------- 可配置项（环境变量可覆盖） ----------
|
||||
: "${ALERT_STATE_FILE:=/root/rust/scripts/.alert-state}"
|
||||
: "${ALERT_COOLDOWN_HOURS:=6}" # 同一级别再次提醒的冷却时间
|
||||
: "${DB_CONTAINER:=1Panel-postgresql-FtMo}"
|
||||
: "${DB_NAME:=milkydata}"
|
||||
: "${DB_USER:=milkydata}"
|
||||
: "${STUCK_ORDER_MINUTES:=30}" # pending 超过该时长视为卡单
|
||||
: "${BACKUP_DIR:=/root/rust/backups/milkydata}" # 备份目录
|
||||
|
||||
DRY_RUN=false
|
||||
[ "${1:-}" = "--dry-run" ] && DRY_RUN=true
|
||||
|
||||
# 日志
|
||||
alert_log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "${MAIL_LOG_DIR}/alert.log" 2>/dev/null || true; }
|
||||
[ "$DRY_RUN" = true ] && alert_log() { echo "[DRY-RUN] $*"; }
|
||||
|
||||
# ---------- 检测项 ----------
|
||||
# 每项输出: 级别(0/1/2)|键名|标签|当前值|详情
|
||||
# 级别: 0=正常 1=警告 2=严重
|
||||
|
||||
check_disk() {
|
||||
local pct; pct=$(df -P / | awk 'NR==2{print $5}' | tr -d '%')
|
||||
pct=${pct:-0}
|
||||
local used total; used=$(df -h / | awk 'NR==2{print $3}'); total=$(df -h / | awk 'NR==2{print $2}')
|
||||
local lvl=0
|
||||
[ "$pct" -ge 90 ] && lvl=2
|
||||
[ "$pct" -ge 80 ] && [ "$lvl" -eq 0 ] && lvl=1
|
||||
echo "${lvl}|disk|磁盘使用|${used} / ${total} (${pct}%)|根分区已用 ${pct}%"
|
||||
}
|
||||
|
||||
check_mem() {
|
||||
local pct; pct=$(free | awk '/Mem/{printf "%.0f", $3/$2 * 100}')
|
||||
pct=${pct:-0}
|
||||
local used total; used=$(free -h | awk '/Mem/{print $3}'); total=$(free -h | awk '/Mem/{print $2}')
|
||||
local lvl=0
|
||||
[ "$pct" -ge 90 ] && lvl=2
|
||||
[ "$pct" -ge 80 ] && [ "$lvl" -eq 0 ] && lvl=1
|
||||
echo "${lvl}|mem|内存使用|${used} / ${total} (${pct}%)|内存已用 ${pct}%"
|
||||
}
|
||||
|
||||
check_backup() {
|
||||
local dir="$BACKUP_DIR"
|
||||
local last; last=$(ls -1t "$dir" 2>/dev/null | head -1)
|
||||
if [ -z "$last" ]; then
|
||||
echo "2|backup|备份|无备份文件|目录 ${dir} 无任何备份"
|
||||
return
|
||||
fi
|
||||
local age; age=$(( ($(date +%s) - $(stat -c %Y "$dir/$last" 2>/dev/null || echo 0)) / 3600 ))
|
||||
local lvl=0
|
||||
[ "$age" -ge 48 ] && lvl=2
|
||||
[ "$age" -ge 24 ] && [ "$lvl" -eq 0 ] && lvl=1
|
||||
echo "${lvl}|backup|备份时效|${age} 小时前（${last}）|最新备份距今 ${age} 小时"
|
||||
}
|
||||
|
||||
check_services() {
|
||||
local bad=""
|
||||
for svc in rust-backend-green rust-backend-blue nginx; do
|
||||
local st; st=$(systemctl is-active "$svc" 2>/dev/null || echo "unknown")
|
||||
[ "$st" = "active" ] || bad="${bad}${svc}=${st} "
|
||||
done
|
||||
if [ -n "$bad" ]; then
|
||||
echo "2|services|服务状态|异常服务: ${bad}|systemd 服务未运行"
|
||||
else
|
||||
echo "0|services|服务状态|全部 active|green/blue/nginx 运行正常"
|
||||
fi
|
||||
}
|
||||
|
||||
check_docker() {
|
||||
local out; out=$(docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null)
|
||||
if [ -z "$out" ]; then
|
||||
if docker info >/dev/null 2>&1; then
|
||||
echo "0|docker|Docker 容器|无运行容器|当前无容器在运行"
|
||||
else
|
||||
echo "2|docker|Docker|不可用|docker 命令执行失败"
|
||||
fi
|
||||
return
|
||||
fi
|
||||
local bad; bad=$(echo "$out" | grep -v " Up " || true)
|
||||
if [ -n "$bad" ]; then
|
||||
echo "2|docker|Docker 容器|$(echo "$out" | grep -c " Up ") 个正常|异常容器: $(echo "$bad" | awk '{print $1}' | tr '\n' ' ')"
|
||||
else
|
||||
echo "0|docker|Docker 容器|$(echo "$out" | grep -c " Up ") 个运行中|全部正常"
|
||||
fi
|
||||
}
|
||||
|
||||
check_ssl() {
|
||||
local expiry; expiry=$(echo "" | openssl s_client -connect xmclassmate.top:443 -servername xmclassmate.top 2>&1 | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||||
if [ -z "$expiry" ]; then
|
||||
echo "1|ssl|SSL 证书|检测失败|无法获取证书到期时间"
|
||||
return
|
||||
fi
|
||||
local days; days=$(( ($(date -d "$expiry" +%s 2>/dev/null || echo 0) - $(date +%s)) / 86400 ))
|
||||
local lvl=0
|
||||
[ "$days" -lt 7 ] && lvl=2
|
||||
[ "$days" -lt 30 ] && [ "$lvl" -eq 0 ] && lvl=1
|
||||
echo "${lvl}|ssl|SSL 证书|剩余 ${days} 天（${expiry}）|证书将于 ${expiry} 到期"
|
||||
}
|
||||
|
||||
# 支付卡单检测（pending 超过 STUCK_ORDER_MINUTES）
|
||||
check_stuck_orders() {
|
||||
local count orders
|
||||
count=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c \
|
||||
"SELECT count(*) FROM payment_orders WHERE status='pending' AND created_at < NOW() - INTERVAL '${STUCK_ORDER_MINUTES} minutes'" 2>/dev/null)
|
||||
if [ -z "$count" ]; then
|
||||
return # 数据库不可用由 check_db 负责
|
||||
fi
|
||||
if [ "$count" -gt 0 ]; then
|
||||
orders=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c \
|
||||
"SELECT order_no || ' (用户' || user_id || ', ' || to_char(created_at, 'MM-DD HH24:MI') || ')' FROM payment_orders WHERE status='pending' AND created_at < NOW() - INTERVAL '${STUCK_ORDER_MINUTES} minutes' ORDER BY created_at" 2>/dev/null | tr '\n' '; ')
|
||||
echo "1|stuck_orders|支付卡单|${count} 笔 pending 超 ${STUCK_ORDER_MINUTES} 分钟|${orders}"
|
||||
else
|
||||
echo "0|stuck_orders|支付卡单|无|无卡单"
|
||||
fi
|
||||
}
|
||||
|
||||
check_db() {
|
||||
local ok; ok=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c "SELECT 1" 2>/dev/null)
|
||||
if [ -z "$ok" ]; then
|
||||
echo "2|db|数据库|不可用|docker exec psql 失败（容器 ${DB_CONTAINER}）"
|
||||
else
|
||||
echo "0|db|数据库|正常|SELECT 1 成功"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------- 状态文件管理 ----------
|
||||
# 格式: key lvl last_alert_epoch
|
||||
load_state() { [ -f "$ALERT_STATE_FILE" ] && cat "$ALERT_STATE_FILE" || true; }
|
||||
|
||||
save_state() {
|
||||
local tmp="${ALERT_STATE_FILE}.tmp"
|
||||
{ for k in "${!NEW_STATE[@]}"; do echo "$k ${NEW_STATE[$k]}"; done; } > "$tmp"
|
||||
mv "$tmp" "$ALERT_STATE_FILE" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# 读取状态：$1=key, $2=字段序号（1=级别 2=时间）；状态行格式为 "key lvl time"
|
||||
get_state() {
|
||||
local line; line=$(load_state | awk -v k="$1" '$1==k')
|
||||
[ -n "$line" ] || { echo "-1"; return; }
|
||||
echo "$line" | awk -v f="$2" '{print $(f+1)}'
|
||||
}
|
||||
|
||||
# ---------- 主流程 ----------
|
||||
NOW=$(date +%s)
|
||||
COOLDOWN_SEC=$((ALERT_COOLDOWN_HOURS * 3600))
|
||||
|
||||
# 收集检测结果（格式: level|key|label|value|detail）
|
||||
RESULTS=""
|
||||
for check in check_disk check_mem check_backup check_services check_docker check_ssl check_db check_stuck_orders; do
|
||||
r=$($check 2>/dev/null)
|
||||
[ -n "$r" ] && RESULTS="${RESULTS}${r}\n"
|
||||
done
|
||||
|
||||
declare -A NEW_STATE
|
||||
ALERTS="" # 需要发邮件的条目
|
||||
WARN_COUNT=0
|
||||
CRIT_COUNT=0
|
||||
|
||||
while IFS='|' read -r lvl key label value detail; do
|
||||
[ -z "$key" ] && continue
|
||||
[ "$lvl" -eq 0 ] && continue # 正常项不处理（下面统一清除状态）
|
||||
|
||||
old_lvl=$(get_state "$key" 1)
|
||||
old_time=$(get_state "$key" 2)
|
||||
[ "$old_time" = "-1" ] && old_time=0
|
||||
|
||||
should_alert=false
|
||||
if [ "$lvl" -gt "$old_lvl" ]; then
|
||||
should_alert=true # 级别上升 → 立即告警
|
||||
elif [ "$lvl" -eq "$old_lvl" ] && [ $((NOW - old_time)) -ge "$COOLDOWN_SEC" ]; then
|
||||
should_alert=true # 同级别冷却期后再次提醒
|
||||
fi
|
||||
|
||||
NEW_STATE[$key]="${lvl} ${NOW}"
|
||||
if [ "$should_alert" = true ]; then
|
||||
[ "$lvl" -eq 2 ] && CRIT_COUNT=$((CRIT_COUNT + 1))
|
||||
[ "$lvl" -eq 1 ] && WARN_COUNT=$((WARN_COUNT + 1))
|
||||
ALERTS="${ALERTS}<tr><td style='padding:8px 10px;border-bottom:1px solid #e2e8f0'>$( [ "$lvl" -eq 2 ] && echo "<span style='color:#dc2626'>严重</span>" || echo "<span style='color:#d97706'>警告</span>" )</td><td style='padding:8px 10px;border-bottom:1px solid #e2e8f0'>${label}</td><td style='padding:8px 10px;border-bottom:1px solid #e2e8f0'>${value}</td><td style='padding:8px 10px;border-bottom:1px solid #e2e8f0;color:#475569'>${detail}</td></tr>\n"
|
||||
alert_log "告警 [${key}] 级别=${lvl}: ${label} ${value} — ${detail}"
|
||||
fi
|
||||
done < <(printf "%b" "$RESULTS")
|
||||
|
||||
# 恢复检测：状态文件中存在但本次为 0/未出现的项 → 清除
|
||||
while read -r key old_lvl _; do
|
||||
[ -z "$key" ] && continue
|
||||
cur_lvl=$(printf "%b" "$RESULTS" | awk -F'|' -v k="$key" '$2==k{print $1; exit}')
|
||||
if [ -z "$cur_lvl" ] || [ "$cur_lvl" -eq 0 ]; then
|
||||
unset 'NEW_STATE[$key]' 2>/dev/null || true
|
||||
alert_log "恢复: ${key} 已恢复正常，状态已清除"
|
||||
fi
|
||||
done < <(load_state)
|
||||
|
||||
save_state
|
||||
|
||||
# ---------- 发送告警邮件 ----------
|
||||
if [ -n "$ALERTS" ]; then
|
||||
if [ "$CRIT_COUNT" -gt 0 ]; then
|
||||
SUBJECT="[ASD 告警] ${CRIT_COUNT} 项严重 / ${WARN_COUNT} 项警告 — $(date '+%m-%d %H:%M')"
|
||||
else
|
||||
SUBJECT="[ASD 告警] ${WARN_COUNT} 项警告 — $(date '+%m-%d %H:%M')"
|
||||
fi
|
||||
BODY=$(cat <<EOF
|
||||
<html><body style="font-family:-apple-system,'PingFang SC','Microsoft YaHei',sans-serif;max-width:640px;margin:0 auto;padding:16px">
|
||||
<h2 style="color:#0f172a;margin:0 0 4px">⚠️ ASD 服务器异常告警</h2>
|
||||
<div style="color:#94a3b8;font-size:13px;margin-bottom:16px">$(date '+%Y-%m-%d %H:%M:%S') · $(hostname)</div>
|
||||
<table style="border-collapse:collapse;width:100%;font-size:13px">
|
||||
<tr><th style="padding:8px 10px;background:#f1f5f9;text-align:left;color:#64748b">级别</th><th style="padding:8px 10px;background:#f1f5f9;text-align:left;color:#64748b">项目</th><th style="padding:8px 10px;background:#f1f5f9;text-align:left;color:#64748b">当前值</th><th style="padding:8px 10px;background:#f1f5f9;text-align:left;color:#64748b">说明</th></tr>
|
||||
$(printf "%b" "$ALERTS")
|
||||
</table>
|
||||
<div style="color:#94a3b8;font-size:12px;margin-top:16px">下次同类提醒将在 ${ALERT_COOLDOWN_HOURS} 小时后，恢复正常后不再打扰。</div>
|
||||
</body></html>
|
||||
EOF
|
||||
)
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
echo "---- 将发送告警邮件 ----"; echo "主题: ${SUBJECT}"; echo "$BODY" | head -30
|
||||
else
|
||||
if mail_send "$SUBJECT" "$BODY"; then
|
||||
alert_log "告警邮件已发送: ${SUBJECT}"
|
||||
else
|
||||
alert_log "告警邮件发送失败: ${SUBJECT}"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
alert_log "本次检查无异常（$(date '+%H:%M:%S')）"
|
||||
[ "$DRY_RUN" = true ] && echo "当前无异常，无需告警。"
|
||||
fi
|
||||
@@ -1,116 +1,207 @@
|
||||
#!/bin/bash
|
||||
# 每日健康报告 — 发送格式化摘要邮件到管理员
|
||||
# 部署：cp 到服务器 /root/rust/scripts/daily-report.sh，cron 每日执行
|
||||
# ===========================================
|
||||
# 每日健康报告 — HTML 摘要邮件（含支付/会员统计）
|
||||
#
|
||||
# 部署：
|
||||
# cp scripts/mail-lib.sh scripts/daily-report.sh → 服务器 /root/rust/scripts/
|
||||
# chmod +x /root/rust/scripts/daily-report.sh
|
||||
# cron（root）:
|
||||
# 0 8 * * * /root/rust/scripts/daily-report.sh >> /root/rust/scripts/logs/daily-report.log 2>&1
|
||||
#
|
||||
# 说明：单项采集失败自动降级为 N/A，不会阻断整封邮件；
|
||||
# 发送失败自动重试并记录 /root/rust/scripts/logs/mail.log。
|
||||
# ===========================================
|
||||
|
||||
set -e
|
||||
# 不用 set -e：采集失败不应导致邮件发不出去
|
||||
set -u
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=mail-lib.sh
|
||||
source "${SCRIPT_DIR}/mail-lib.sh"
|
||||
|
||||
# ---------- 可配置项 ----------
|
||||
: "${DB_CONTAINER:=1Panel-postgresql-FtMo}" # PostgreSQL 容器名
|
||||
: "${DB_NAME:=milkydata}" # 数据库名
|
||||
: "${DB_USER:=milkydata}" # 数据库用户
|
||||
: "${BACKUP_DIR:=/root/rust/backups/milkydata}" # 备份目录
|
||||
|
||||
TO="m943790568@163.com"
|
||||
DATE=$(date "+%Y-%m-%d")
|
||||
HOST=$(hostname)
|
||||
SUBJECT="[ASD 服务器报告] ${DATE} — ${HOST}"
|
||||
|
||||
# ---------- 采集数据 ----------
|
||||
# ---------- 状态徽章 ----------
|
||||
# badge <ok|warn|crit> <文本>
|
||||
badge() {
|
||||
case "$1" in
|
||||
ok) echo "<span style='background:#16a34a;color:#fff;padding:1px 8px;border-radius:10px;font-size:12px'>$2</span>" ;;
|
||||
warn) echo "<span style='background:#d97706;color:#fff;padding:1px 8px;border-radius:10px;font-size:12px'>$2</span>" ;;
|
||||
crit) echo "<span style='background:#dc2626;color:#fff;padding:1px 8px;border-radius:10px;font-size:12px'>$2</span>" ;;
|
||||
*) echo "<span style='background:#64748b;color:#fff;padding:1px 8px;border-radius:10px;font-size:12px'>$2</span>" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# 根据磁盘/内存百分比判定级别（参数: 数值，返回 ok/warn/crit/unknown）
|
||||
pct_level() {
|
||||
local v="$1" w="${2:-80}" c="${3:-90}"
|
||||
case "$v" in
|
||||
''|*[!0-9]*) echo "unknown"; return ;;
|
||||
esac
|
||||
if [ "$v" -ge "$c" ]; then echo "crit"
|
||||
elif [ "$v" -ge "$w" ]; then echo "warn"
|
||||
else echo "ok"; fi
|
||||
}
|
||||
|
||||
# ---------- 系统状态 ----------
|
||||
|
||||
UPTIME=$(uptime -p | sed 's/up //')
|
||||
LOAD=$(uptime | awk -F'load average:' '{print $2}' | xargs)
|
||||
LOAD1=${LOAD%%,*} # 1 分钟负载（可能是小数，取整数部分比较）
|
||||
LOAD1_INT=${LOAD1%%.*}
|
||||
DISK_USED=$(df -h / | awk 'NR==2{print $3}')
|
||||
DISK_TOTAL=$(df -h / | awk 'NR==2{print $2}')
|
||||
DISK_PCT=$(df -h / | awk 'NR==2{print $5}')
|
||||
DISK_PCT_STR=$(df -h / | awk 'NR==2{print $5}')
|
||||
DISK_PCT=${DISK_PCT_STR%\%}
|
||||
MEM_USED=$(free -h | awk '/Mem/{print $3}')
|
||||
MEM_TOTAL=$(free -h | awk '/Mem/{print $2}')
|
||||
MEM_PCT=$(free | awk '/Mem/{printf "%.0f", $3/$2 * 100}')
|
||||
CPU_TEMP=$(cat /sys/class/thermal/thermal_zone0/temp 2>/dev/null | awk '{printf "%.1f°C", $1/1000}' || echo "N/A")
|
||||
|
||||
# Docker
|
||||
DOCKER_STATUS=$(docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null || echo "Docker 不可用")
|
||||
# ---------- Docker ----------
|
||||
DOCKER_STATUS=$(docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null || echo "")
|
||||
DOCKER_DOWN=$(echo "$DOCKER_STATUS" | grep -vc "Up " || true)
|
||||
if [ -z "$DOCKER_STATUS" ]; then
|
||||
DOCKER_BODY="<span style='color:#dc2626'>Docker 不可用或无运行容器</span>"
|
||||
elif [ "$DOCKER_DOWN" -gt 0 ]; then
|
||||
DOCKER_BODY="$(echo "$DOCKER_STATUS" | awk '{print $1": "$2}' | sed 's/^/ /' | sed 's/&/\&/g;s/</\</g;s/>/\>/g' | tr '\n' '<br>')"
|
||||
else
|
||||
DOCKER_BODY="$(echo "$DOCKER_STATUS" | awk '{print $1" "$2}' | sed 's/&/\&/g;s/</\</g;s/>/\>/g' | tr '\n' '<br>')"
|
||||
fi
|
||||
|
||||
# SSL 证书
|
||||
# ---------- SSL 证书 ----------
|
||||
SSL_EXPIRY=$(echo "" | openssl s_client -connect xmclassmate.top:443 -servername xmclassmate.top 2>&1 | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2 || echo "检测失败")
|
||||
SSL_DAYS=$(( ($(date -d "$SSL_EXPIRY" +%s 2>/dev/null) - $(date +%s)) / 86400 )) || SSL_DAYS="?"
|
||||
if [ "$SSL_EXPIRY" != "检测失败" ]; then
|
||||
SSL_DAYS=$(( ($(date -d "$SSL_EXPIRY" +%s 2>/dev/null) - $(date +%s)) / 86400 ))
|
||||
else
|
||||
SSL_DAYS=9999
|
||||
fi
|
||||
|
||||
# 服务状态
|
||||
# ---------- 服务状态 ----------
|
||||
GREEN_STATUS=$(systemctl is-active rust-backend-green.service 2>/dev/null || echo "unknown")
|
||||
BLUE_STATUS=$(systemctl is-active rust-backend-blue.service 2>/dev/null || echo "unknown")
|
||||
NGINX_STATUS=$(systemctl is-active nginx 2>/dev/null || echo "unknown")
|
||||
|
||||
# 最近备份
|
||||
LAST_BACKUP=$(ls -1t /root/rust/backups/milkydata/ 2>/dev/null | head -1)
|
||||
BACKUP_AGE="未知"
|
||||
# ---------- 备份 ----------
|
||||
LAST_BACKUP=$(ls -1t "$BACKUP_DIR/" 2>/dev/null | head -1)
|
||||
BACKUP_AGE_HOURS=""
|
||||
BACKUP_AGE_TEXT="未知"
|
||||
if [ -n "$LAST_BACKUP" ]; then
|
||||
BACKUP_TIME=$(stat -c %Y "/root/rust/backups/milkydata/$LAST_BACKUP" 2>/dev/null)
|
||||
BACKUP_TIME=$(stat -c %Y "$BACKUP_DIR/$LAST_BACKUP" 2>/dev/null)
|
||||
NOW=$(date +%s)
|
||||
AGE_HOURS=$(( (NOW - BACKUP_TIME) / 3600 ))
|
||||
if [ "$AGE_HOURS" -lt 24 ]; then
|
||||
BACKUP_AGE="✅ ${AGE_HOURS}小时前"
|
||||
elif [ "$AGE_HOURS" -lt 48 ]; then
|
||||
BACKUP_AGE="⚠️ ${AGE_HOURS}小时前"
|
||||
else
|
||||
BACKUP_AGE="❌ ${AGE_HOURS}小时前"
|
||||
fi
|
||||
BACKUP_AGE_HOURS=$(( (NOW - BACKUP_TIME) / 3600 ))
|
||||
BACKUP_AGE_TEXT="${BACKUP_AGE_HOURS} 小时前"
|
||||
fi
|
||||
|
||||
# Fail2Ban 封禁统计
|
||||
# ---------- 安全 ----------
|
||||
SSH_BANS=$(sudo fail2ban-client status sshd 2>/dev/null | grep "Currently banned" | awk '{print $NF}' || echo "0")
|
||||
NGINX_BANS=$(sudo fail2ban-client status nginx-botsearch 2>/dev/null | grep "Currently banned" | awk '{print $NF}' || echo "0")
|
||||
|
||||
# 近期错误数（昨天 journalctl）
|
||||
YESTERDAY_ERRORS=$(journalctl -u rust-backend-green.service --since "1 day ago" --priority err --no-pager 2>/dev/null | wc -l)
|
||||
YESTERDAY_WARNS=$(journalctl -u rust-backend-green.service --since "1 day ago" --priority warning --no-pager 2>/dev/null | wc -l)
|
||||
|
||||
# 应用统计（昨天活跃用户）
|
||||
# ---------- 应用概况（昨日） ----------
|
||||
ACTIVE_USERS=$(journalctl -u rust-backend-green.service --since "1 day ago" --no-pager 2>/dev/null | grep -c "user_id=\|登录成功\|上传成功" || echo "0")
|
||||
API_COUNT=$(journalctl -u rust-backend-green.service --since "1 day ago" --no-pager 2>/dev/null | grep -c "INFO\|WARN\|ERROR" || echo "0")
|
||||
|
||||
# ---------- 构建邮件正文 ----------
|
||||
# ---------- 支付与会员（数据库统计，不可用则 N/A） ----------
|
||||
# 服务器上通过 docker exec 访问 PostgreSQL（与 check-members.sh 同模式）
|
||||
DB_QUERY() {
|
||||
docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c "$1" 2>/dev/null || echo ""
|
||||
}
|
||||
DB_OK=false
|
||||
if [ -n "$(DB_QUERY 'SELECT 1')" ]; then DB_OK=true; fi
|
||||
|
||||
SUBJECT="[ASD 服务器报告] ${DATE} — ${HOST}"
|
||||
PAY_ORDERS_TODAY="N/A"; PAY_PAID_TODAY="N/A"; PAY_STUCK="N/A"
|
||||
MEM_ACTIVE="N/A"; MEM_EXPIRE_7D="N/A"
|
||||
if [ "$DB_OK" = true ]; then
|
||||
PAY_ORDERS_TODAY=$(DB_QUERY "SELECT count(*) FROM payment_orders WHERE created_at::date = CURRENT_DATE")
|
||||
PAY_PAID_TODAY=$(DB_QUERY "SELECT count(*) FROM payment_orders WHERE status='paid' AND paid_at::date = CURRENT_DATE")
|
||||
PAY_STUCK=$(DB_QUERY "SELECT count(*) FROM payment_orders WHERE status='pending' AND created_at < NOW() - INTERVAL '30 minutes'")
|
||||
MEM_ACTIVE=$(DB_QUERY "SELECT count(*) FROM users WHERE is_member AND (membership_expires_at IS NULL OR membership_expires_at > NOW())")
|
||||
MEM_EXPIRE_7D=$(DB_QUERY "SELECT count(*) FROM users WHERE is_member AND membership_expires_at IS NOT NULL AND membership_expires_at BETWEEN NOW() AND NOW() + INTERVAL '7 days'")
|
||||
fi
|
||||
[ -z "$PAY_ORDERS_TODAY" ] && PAY_ORDERS_TODAY="N/A"
|
||||
[ -z "$PAY_PAID_TODAY" ] && PAY_PAID_TODAY="N/A"
|
||||
[ -z "$PAY_STUCK" ] && PAY_STUCK="N/A"
|
||||
[ -z "$MEM_ACTIVE" ] && MEM_ACTIVE="N/A"
|
||||
[ -z "$MEM_EXPIRE_7D" ] && MEM_EXPIRE_7D="N/A"
|
||||
|
||||
MAIL=$(cat <<EOF
|
||||
===========================================
|
||||
ASD 服务器健康报告
|
||||
${DATE}
|
||||
主机: ${HOST}
|
||||
===========================================
|
||||
# ---------- 组装 HTML ----------
|
||||
|
||||
🖥 系统状态
|
||||
- 运行时长: ${UPTIME}
|
||||
- 负载: ${LOAD}
|
||||
- 磁盘: ${DISK_USED} / ${DISK_TOTAL} (${DISK_PCT})
|
||||
- 内存: ${MEM_USED} / ${MEM_TOTAL} (${MEM_PCT})
|
||||
- CPU 温度: ${CPU_TEMP}
|
||||
# 行生成器: row <指标> <当前值> <级别> <附加说明?>
|
||||
row() {
|
||||
echo "<tr><td style='padding:6px 10px;border-bottom:1px solid #e2e8f0;color:#475569'>$1</td><td style='padding:6px 10px;border-bottom:1px solid #e2e8f0'>$2</td><td style='padding:6px 10px;border-bottom:1px solid #e2e8f0;text-align:center'>$(badge "$3" "$4")</td></tr>"
|
||||
}
|
||||
section_title() {
|
||||
echo "<tr><td colspan='3' style='padding:10px;background:#f8fafc;font-weight:bold;color:#0f172a;border-bottom:1px solid #e2e8f0'>$1</td></tr>"
|
||||
}
|
||||
|
||||
📦 容器
|
||||
$(echo "$DOCKER_STATUS" | sed 's/^/ /')
|
||||
STYLE="font-family:-apple-system,'PingFang SC','Microsoft YaHei',sans-serif;background:#fff;max-width:640px;margin:0 auto;padding:16px"
|
||||
TH="padding:8px 10px;background:#f1f5f9;color:#64748b;text-align:left;font-size:12px;border-bottom:2px solid #e2e8f0"
|
||||
|
||||
🔐 SSL 证书
|
||||
域名: xmclassmate.top
|
||||
到期: ${SSL_EXPIRY}
|
||||
剩余: ${SSL_DAYS} 天
|
||||
BODY=$(cat <<EOF
|
||||
<html><body style="${STYLE}">
|
||||
<h2 style="color:#0f172a;margin:0 0 4px">ASD 服务器健康报告</h2>
|
||||
<div style="color:#94a3b8;font-size:13px;margin-bottom:16px">${DATE} · ${HOST} · 发送于 $(date "+%H:%M")</div>
|
||||
|
||||
🔄 服务状态
|
||||
- 后端 (green): ${GREEN_STATUS}
|
||||
- 后端 (blue): ${BLUE_STATUS}
|
||||
- nginx: ${NGINX_STATUS}
|
||||
<table style="border-collapse:collapse;width:100%;font-size:13px">
|
||||
<tr><th style="${TH}">指标</th><th style="${TH}">当前值</th><th style="${TH}" width="90">状态</th></tr>
|
||||
|
||||
🛡 安全
|
||||
- SSH 封禁: ${SSH_BANS} 个 IP
|
||||
- nginx 封禁: ${NGINX_BANS} 个 IP
|
||||
- 昨日错误: ${YESTERDAY_ERRORS}
|
||||
- 昨日警告: ${YESTERDAY_WARNS}
|
||||
$(section_title "🖥 系统状态")
|
||||
$(row "运行时长" "$UPTIME" ok "OK")
|
||||
$(row "系统负载" "$LOAD" "$([ "$LOAD1_INT" -ge 4 ] && echo crit || { [ "$LOAD1_INT" -ge 2 ] && echo warn || echo ok; })" "$([ "$LOAD1_INT" -ge 4 ] && echo "超载" || { [ "$LOAD1_INT" -ge 2 ] && echo "偏高" || echo "正常"; })")
|
||||
$(row "磁盘使用" "${DISK_USED} / ${DISK_TOTAL} (${DISK_PCT_STR})" "$(pct_level "$DISK_PCT")" "$(pct_level "$DISK_PCT" | tr 'a-z' 'A-Z')")
|
||||
$(row "内存使用" "${MEM_USED} / ${MEM_TOTAL} (${MEM_PCT}%)" "$(pct_level "$MEM_PCT" 80 90)" "$(pct_level "$MEM_PCT" 80 90 | tr 'a-z' 'A-Z')")
|
||||
$(row "CPU 温度" "${CPU_TEMP}" ok "OK")
|
||||
|
||||
📊 应用概况（昨日）
|
||||
- 日志事件数: ${API_COUNT}
|
||||
- 活跃用户操作: ${ACTIVE_USERS}
|
||||
$(section_title "📦 容器与服务")
|
||||
$(row "Docker 容器" "运行中 $(echo "$DOCKER_STATUS" | grep -c "Up " || echo 0) 个 / 异常 $(echo "$DOCKER_STATUS" | grep -vc "Up " || echo 0) 个" "$([ "$DOCKER_DOWN" -gt 0 ] && echo crit || echo ok)" "$([ "$DOCKER_DOWN" -gt 0 ] && echo "有异常" || echo "正常")")
|
||||
$(row "后端 green" "$GREEN_STATUS" "$([ "$GREEN_STATUS" = "active" ] && echo ok || echo crit)" "$([ "$GREEN_STATUS" = "active" ] && echo "运行中" || echo "异常")")
|
||||
$(row "后端 blue" "$BLUE_STATUS" "$([ "$BLUE_STATUS" = "active" ] && echo ok || echo crit)" "$([ "$BLUE_STATUS" = "active" ] && echo "运行中" || echo "异常")")
|
||||
$(row "nginx" "$NGINX_STATUS" "$([ "$NGINX_STATUS" = "active" ] && echo ok || echo crit)" "$([ "$NGINX_STATUS" = "active" ] && echo "运行中" || echo "异常")")
|
||||
<tr><td colspan="3" style="padding:6px 10px;color:#64748b;font-size:12px;background:#fbfdff">${DOCKER_BODY}</td></tr>
|
||||
|
||||
💾 备份
|
||||
- 最新备份: ${LAST_BACKUP:-无}
|
||||
- 备份时效: ${BACKUP_AGE}
|
||||
$(section_title "🔐 SSL 与安全")
|
||||
$(row "SSL 证书" "${SSL_EXPIRY}（剩余 ${SSL_DAYS} 天）" "$([ "$SSL_DAYS" -lt 7 ] && echo crit || { [ "$SSL_DAYS" -lt 30 ] && echo warn || echo ok; })" "$([ "$SSL_DAYS" -lt 7 ] && echo "即将过期" || { [ "$SSL_DAYS" -lt 30 ] && echo "注意续期" || echo "正常"; })")
|
||||
$(row "Fail2ban SSH" "${SSH_BANS} 个 IP 被封禁" ok "OK")
|
||||
$(row "Fail2ban nginx" "${NGINX_BANS} 个 IP 被封禁" ok "OK")
|
||||
$(row "昨日日志错误" "$YESTERDAY_ERRORS 条" "$([ "$YESTERDAY_ERRORS" -gt 50 ] && echo warn || echo ok)" "$([ "$YESTERDAY_ERRORS" -gt 50 ] && echo "偏多" || echo "正常")")
|
||||
$(row "昨日日志警告" "$YESTERDAY_WARNS 条" "$([ "$YESTERDAY_WARNS" -gt 200 ] && echo warn || echo ok)" "$([ "$YESTERDAY_WARNS" -gt 200 ] && echo "偏多" || echo "正常")")
|
||||
|
||||
---
|
||||
ASD 自动监控 | $(date "+%Y-%m-%d %H:%M")
|
||||
$(section_title "💾 备份")
|
||||
$(row "最新备份" "${LAST_BACKUP:-无}" "$([ -n "$BACKUP_AGE_HOURS" ] && { [ "$BACKUP_AGE_HOURS" -ge 48 ] && echo crit || { [ "$BACKUP_AGE_HOURS" -ge 24 ] && echo warn || echo ok; }; } || echo crit)" "$([ -n "$BACKUP_AGE_HOURS" ] && { [ "$BACKUP_AGE_HOURS" -ge 48 ] && echo "严重过期" || { [ "$BACKUP_AGE_HOURS" -ge 24 ] && echo "⚠ 超24小时" || echo "${BACKUP_AGE_TEXT}"; }; } || echo "未知")")
|
||||
$(row "备份时效" "${BACKUP_AGE_TEXT}" ok "OK")
|
||||
|
||||
$(section_title "📊 应用概况（昨日）")
|
||||
$(row "日志事件数" "$API_COUNT 条" ok "OK")
|
||||
$(row "活跃用户操作" "$ACTIVE_USERS 次" ok "OK")
|
||||
|
||||
$(section_title "💰 支付与会员")
|
||||
$(row "今日新订单" "$PAY_ORDERS_TODAY 笔" ok "OK")
|
||||
$(row "今日支付成功" "$PAY_PAID_TODAY 笔" ok "OK")
|
||||
$(row "卡单（pending>30分钟）" "$PAY_STUCK 笔" "$([ "$PAY_STUCK" -gt 0 ] 2>/dev/null && echo crit || echo ok)" "$([ "$PAY_STUCK" -gt 0 ] 2>/dev/null && echo "需人工处理" || echo "无")")
|
||||
$(row "活跃会员" "$MEM_ACTIVE 人" ok "OK")
|
||||
$(row "7天内到期会员" "$MEM_EXPIRE_7D 人" "$([ "$MEM_EXPIRE_7D" -gt 0 ] 2>/dev/null && echo warn || echo ok)" "$([ "$MEM_EXPIRE_7D" -gt 0 ] 2>/dev/null && echo "可发续费提醒" || echo "无")")
|
||||
$( [ "$DB_OK" = false ] && echo "<tr><td colspan='3' style='padding:6px 10px;color:#dc2626;font-size:12px;background:#fef2f2'>⚠ 数据库不可用，支付/会员统计缺失</td></tr>" )
|
||||
</table>
|
||||
|
||||
<div style="color:#94a3b8;font-size:12px;margin-top:16px">ASD 自动监控 | $(date "+%Y-%m-%d %H:%M") | 异常项请关注 alert.sh 实时告警</div>
|
||||
</body></html>
|
||||
EOF
|
||||
)
|
||||
|
||||
# ---------- 发送邮件 ----------
|
||||
|
||||
echo "$MAIL" | msmtp -a default "$TO"
|
||||
echo "✅ 健康报告已发送到 ${TO}"
|
||||
# ---------- 发送 ----------
|
||||
if mail_send "$SUBJECT" "$BODY"; then
|
||||
echo "✅ 健康报告已发送到 ${MAIL_TO}（$(date '+%H:%M:%S')）"
|
||||
else
|
||||
echo "❌ 健康报告发送失败，详见 ${MAIL_LOG_DIR}/mail.log"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
70
scripts/mail-lib.sh
Normal file
70
scripts/mail-lib.sh
Normal file
@@ -0,0 +1,70 @@
|
||||
#!/bin/bash
|
||||
# ===========================================
|
||||
# mail-lib.sh — 邮件发送共享库
|
||||
# 供 daily-report.sh / alert.sh 复用：
|
||||
# mail_send <主题> <HTML正文> （带重试 + 日志 + 轮转）
|
||||
# ===========================================
|
||||
|
||||
# ---------- 可配置项（环境变量可覆盖） ----------
|
||||
: "${MAIL_TO:=m943790568@163.com}" # 收件人，多个用逗号分隔
|
||||
: "${MAIL_FROM:=ASD 监控 <m943790568@163.com>}" # 发件人显示名
|
||||
: "${MAIL_ACCOUNT:=default}" # msmtp 账户（~/.msmtprc 中的 account）
|
||||
: "${MAIL_LOG_DIR:=/root/rust/scripts/logs}" # 日志目录
|
||||
: "${MAIL_RETRY:=3}" # 发送重试次数
|
||||
: "${MAIL_LOG_MAX_KB:=1024}" # mail.log 轮转阈值（KB）
|
||||
|
||||
mkdir -p "$MAIL_LOG_DIR" 2>/dev/null || true
|
||||
|
||||
# 写日志
|
||||
mail_log() {
|
||||
local level="$1"; shift
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [${level}] $*" >> "${MAIL_LOG_DIR}/mail.log" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# 日志轮转：超过阈值保留末尾 200 行
|
||||
mail_rotate_log() {
|
||||
local f="${MAIL_LOG_DIR}/mail.log"
|
||||
[ -f "$f" ] || return 0
|
||||
local size_kb=$(( $(stat -c %s "$f" 2>/dev/null || echo 0) / 1024 ))
|
||||
if [ "$size_kb" -gt "$MAIL_LOG_MAX_KB" ]; then
|
||||
tail -n 200 "$f" > "$f.tmp" 2>/dev/null && mv "$f.tmp" "$f" 2>/dev/null || true
|
||||
mail_log INFO "mail.log 已轮转"
|
||||
fi
|
||||
}
|
||||
|
||||
# 发送 HTML 邮件（UTF-8），失败自动重试
|
||||
# 用法: mail_send "主题" "HTML 正文"
|
||||
mail_send() {
|
||||
local subject="$1" body="$2"
|
||||
mail_rotate_log
|
||||
|
||||
local recipients headers
|
||||
recipients=$(echo "$MAIL_TO" | tr ',' ' ')
|
||||
headers=$(cat <<EOF
|
||||
From: ${MAIL_FROM}
|
||||
To: ${MAIL_TO}
|
||||
Subject: ${subject}
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/html; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
EOF
|
||||
)
|
||||
|
||||
local attempt=1 ok=0
|
||||
while [ "$attempt" -le "$MAIL_RETRY" ]; do
|
||||
if { echo "$headers"; echo; echo "$body"; } | msmtp -a "$MAIL_ACCOUNT" $recipients 2>>"${MAIL_LOG_DIR}/mail.log"; then
|
||||
ok=1
|
||||
mail_log INFO "邮件发送成功: ${subject} → ${MAIL_TO}（第 ${attempt} 次尝试）"
|
||||
break
|
||||
fi
|
||||
mail_log WARN "邮件发送失败（第 ${attempt}/${MAIL_RETRY} 次）: ${subject}"
|
||||
attempt=$((attempt + 1))
|
||||
[ "$attempt" -le "$MAIL_RETRY" ] && sleep $((attempt * 5))
|
||||
done
|
||||
|
||||
if [ "$ok" -ne 1 ]; then
|
||||
mail_log ERROR "邮件最终发送失败: ${subject}"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
Reference in New Issue
Block a user