feat: 邮件告警与每日报告优化（HTML+UTF-8、阈值实时告警、支付卡单检测、重试与状态机）
Some checks failed
Deploy Backend / deploy (push) Has been cancelled
Some checks failed
Deploy Backend / deploy (push) Has been cancelled
This commit is contained in:
@@ -1,116 +1,207 @@
|
||||
#!/bin/bash
|
||||
# 每日健康报告 — 发送格式化摘要邮件到管理员
|
||||
# 部署：cp 到服务器 /root/rust/scripts/daily-report.sh，cron 每日执行
|
||||
# ===========================================
|
||||
# 每日健康报告 — HTML 摘要邮件（含支付/会员统计）
|
||||
#
|
||||
# 部署：
|
||||
# cp scripts/mail-lib.sh scripts/daily-report.sh → 服务器 /root/rust/scripts/
|
||||
# chmod +x /root/rust/scripts/daily-report.sh
|
||||
# cron（root）:
|
||||
# 0 8 * * * /root/rust/scripts/daily-report.sh >> /root/rust/scripts/logs/daily-report.log 2>&1
|
||||
#
|
||||
# 说明：单项采集失败自动降级为 N/A，不会阻断整封邮件；
|
||||
# 发送失败自动重试并记录 /root/rust/scripts/logs/mail.log。
|
||||
# ===========================================
|
||||
|
||||
set -e
|
||||
# 不用 set -e：采集失败不应导致邮件发不出去
|
||||
set -u
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=mail-lib.sh
|
||||
source "${SCRIPT_DIR}/mail-lib.sh"
|
||||
|
||||
# ---------- 可配置项 ----------
|
||||
: "${DB_CONTAINER:=1Panel-postgresql-FtMo}" # PostgreSQL 容器名
|
||||
: "${DB_NAME:=milkydata}" # 数据库名
|
||||
: "${DB_USER:=milkydata}" # 数据库用户
|
||||
: "${BACKUP_DIR:=/root/rust/backups/milkydata}" # 备份目录
|
||||
|
||||
TO="m943790568@163.com"
|
||||
DATE=$(date "+%Y-%m-%d")
|
||||
HOST=$(hostname)
|
||||
SUBJECT="[ASD 服务器报告] ${DATE} — ${HOST}"
|
||||
|
||||
# ---------- 采集数据 ----------
|
||||
# ---------- 状态徽章 ----------
|
||||
# badge <ok|warn|crit> <文本>
|
||||
badge() {
|
||||
case "$1" in
|
||||
ok) echo "<span style='background:#16a34a;color:#fff;padding:1px 8px;border-radius:10px;font-size:12px'>$2</span>" ;;
|
||||
warn) echo "<span style='background:#d97706;color:#fff;padding:1px 8px;border-radius:10px;font-size:12px'>$2</span>" ;;
|
||||
crit) echo "<span style='background:#dc2626;color:#fff;padding:1px 8px;border-radius:10px;font-size:12px'>$2</span>" ;;
|
||||
*) echo "<span style='background:#64748b;color:#fff;padding:1px 8px;border-radius:10px;font-size:12px'>$2</span>" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# 根据磁盘/内存百分比判定级别（参数: 数值，返回 ok/warn/crit/unknown）
|
||||
pct_level() {
|
||||
local v="$1" w="${2:-80}" c="${3:-90}"
|
||||
case "$v" in
|
||||
''|*[!0-9]*) echo "unknown"; return ;;
|
||||
esac
|
||||
if [ "$v" -ge "$c" ]; then echo "crit"
|
||||
elif [ "$v" -ge "$w" ]; then echo "warn"
|
||||
else echo "ok"; fi
|
||||
}
|
||||
|
||||
# ---------- 系统状态 ----------
|
||||
|
||||
UPTIME=$(uptime -p | sed 's/up //')
|
||||
LOAD=$(uptime | awk -F'load average:' '{print $2}' | xargs)
|
||||
LOAD1=${LOAD%%,*} # 1 分钟负载（可能是小数，取整数部分比较）
|
||||
LOAD1_INT=${LOAD1%%.*}
|
||||
DISK_USED=$(df -h / | awk 'NR==2{print $3}')
|
||||
DISK_TOTAL=$(df -h / | awk 'NR==2{print $2}')
|
||||
DISK_PCT=$(df -h / | awk 'NR==2{print $5}')
|
||||
DISK_PCT_STR=$(df -h / | awk 'NR==2{print $5}')
|
||||
DISK_PCT=${DISK_PCT_STR%\%}
|
||||
MEM_USED=$(free -h | awk '/Mem/{print $3}')
|
||||
MEM_TOTAL=$(free -h | awk '/Mem/{print $2}')
|
||||
MEM_PCT=$(free | awk '/Mem/{printf "%.0f", $3/$2 * 100}')
|
||||
CPU_TEMP=$(cat /sys/class/thermal/thermal_zone0/temp 2>/dev/null | awk '{printf "%.1f°C", $1/1000}' || echo "N/A")
|
||||
|
||||
# Docker
|
||||
DOCKER_STATUS=$(docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null || echo "Docker 不可用")
|
||||
# ---------- Docker ----------
|
||||
DOCKER_STATUS=$(docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null || echo "")
|
||||
DOCKER_DOWN=$(echo "$DOCKER_STATUS" | grep -vc "Up " || true)
|
||||
if [ -z "$DOCKER_STATUS" ]; then
|
||||
DOCKER_BODY="<span style='color:#dc2626'>Docker 不可用或无运行容器</span>"
|
||||
elif [ "$DOCKER_DOWN" -gt 0 ]; then
|
||||
DOCKER_BODY="$(echo "$DOCKER_STATUS" | awk '{print $1": "$2}' | sed 's/^/ /' | sed 's/&/\&/g;s/</\</g;s/>/\>/g' | tr '\n' '<br>')"
|
||||
else
|
||||
DOCKER_BODY="$(echo "$DOCKER_STATUS" | awk '{print $1" "$2}' | sed 's/&/\&/g;s/</\</g;s/>/\>/g' | tr '\n' '<br>')"
|
||||
fi
|
||||
|
||||
# SSL 证书
|
||||
# ---------- SSL 证书 ----------
|
||||
SSL_EXPIRY=$(echo "" | openssl s_client -connect xmclassmate.top:443 -servername xmclassmate.top 2>&1 | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2 || echo "检测失败")
|
||||
SSL_DAYS=$(( ($(date -d "$SSL_EXPIRY" +%s 2>/dev/null) - $(date +%s)) / 86400 )) || SSL_DAYS="?"
|
||||
if [ "$SSL_EXPIRY" != "检测失败" ]; then
|
||||
SSL_DAYS=$(( ($(date -d "$SSL_EXPIRY" +%s 2>/dev/null) - $(date +%s)) / 86400 ))
|
||||
else
|
||||
SSL_DAYS=9999
|
||||
fi
|
||||
|
||||
# 服务状态
|
||||
# ---------- 服务状态 ----------
|
||||
GREEN_STATUS=$(systemctl is-active rust-backend-green.service 2>/dev/null || echo "unknown")
|
||||
BLUE_STATUS=$(systemctl is-active rust-backend-blue.service 2>/dev/null || echo "unknown")
|
||||
NGINX_STATUS=$(systemctl is-active nginx 2>/dev/null || echo "unknown")
|
||||
|
||||
# 最近备份
|
||||
LAST_BACKUP=$(ls -1t /root/rust/backups/milkydata/ 2>/dev/null | head -1)
|
||||
BACKUP_AGE="未知"
|
||||
# ---------- 备份 ----------
|
||||
LAST_BACKUP=$(ls -1t "$BACKUP_DIR/" 2>/dev/null | head -1)
|
||||
BACKUP_AGE_HOURS=""
|
||||
BACKUP_AGE_TEXT="未知"
|
||||
if [ -n "$LAST_BACKUP" ]; then
|
||||
BACKUP_TIME=$(stat -c %Y "/root/rust/backups/milkydata/$LAST_BACKUP" 2>/dev/null)
|
||||
BACKUP_TIME=$(stat -c %Y "$BACKUP_DIR/$LAST_BACKUP" 2>/dev/null)
|
||||
NOW=$(date +%s)
|
||||
AGE_HOURS=$(( (NOW - BACKUP_TIME) / 3600 ))
|
||||
if [ "$AGE_HOURS" -lt 24 ]; then
|
||||
BACKUP_AGE="✅ ${AGE_HOURS}小时前"
|
||||
elif [ "$AGE_HOURS" -lt 48 ]; then
|
||||
BACKUP_AGE="⚠️ ${AGE_HOURS}小时前"
|
||||
else
|
||||
BACKUP_AGE="❌ ${AGE_HOURS}小时前"
|
||||
fi
|
||||
BACKUP_AGE_HOURS=$(( (NOW - BACKUP_TIME) / 3600 ))
|
||||
BACKUP_AGE_TEXT="${BACKUP_AGE_HOURS} 小时前"
|
||||
fi
|
||||
|
||||
# Fail2Ban 封禁统计
|
||||
# ---------- 安全 ----------
|
||||
SSH_BANS=$(sudo fail2ban-client status sshd 2>/dev/null | grep "Currently banned" | awk '{print $NF}' || echo "0")
|
||||
NGINX_BANS=$(sudo fail2ban-client status nginx-botsearch 2>/dev/null | grep "Currently banned" | awk '{print $NF}' || echo "0")
|
||||
|
||||
# 近期错误数（昨天 journalctl）
|
||||
YESTERDAY_ERRORS=$(journalctl -u rust-backend-green.service --since "1 day ago" --priority err --no-pager 2>/dev/null | wc -l)
|
||||
YESTERDAY_WARNS=$(journalctl -u rust-backend-green.service --since "1 day ago" --priority warning --no-pager 2>/dev/null | wc -l)
|
||||
|
||||
# 应用统计（昨天活跃用户）
|
||||
# ---------- 应用概况（昨日） ----------
|
||||
ACTIVE_USERS=$(journalctl -u rust-backend-green.service --since "1 day ago" --no-pager 2>/dev/null | grep -c "user_id=\|登录成功\|上传成功" || echo "0")
|
||||
API_COUNT=$(journalctl -u rust-backend-green.service --since "1 day ago" --no-pager 2>/dev/null | grep -c "INFO\|WARN\|ERROR" || echo "0")
|
||||
|
||||
# ---------- 构建邮件正文 ----------
|
||||
# ---------- 支付与会员（数据库统计，不可用则 N/A） ----------
|
||||
# 服务器上通过 docker exec 访问 PostgreSQL（与 check-members.sh 同模式）
|
||||
DB_QUERY() {
|
||||
docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c "$1" 2>/dev/null || echo ""
|
||||
}
|
||||
DB_OK=false
|
||||
if [ -n "$(DB_QUERY 'SELECT 1')" ]; then DB_OK=true; fi
|
||||
|
||||
SUBJECT="[ASD 服务器报告] ${DATE} — ${HOST}"
|
||||
PAY_ORDERS_TODAY="N/A"; PAY_PAID_TODAY="N/A"; PAY_STUCK="N/A"
|
||||
MEM_ACTIVE="N/A"; MEM_EXPIRE_7D="N/A"
|
||||
if [ "$DB_OK" = true ]; then
|
||||
PAY_ORDERS_TODAY=$(DB_QUERY "SELECT count(*) FROM payment_orders WHERE created_at::date = CURRENT_DATE")
|
||||
PAY_PAID_TODAY=$(DB_QUERY "SELECT count(*) FROM payment_orders WHERE status='paid' AND paid_at::date = CURRENT_DATE")
|
||||
PAY_STUCK=$(DB_QUERY "SELECT count(*) FROM payment_orders WHERE status='pending' AND created_at < NOW() - INTERVAL '30 minutes'")
|
||||
MEM_ACTIVE=$(DB_QUERY "SELECT count(*) FROM users WHERE is_member AND (membership_expires_at IS NULL OR membership_expires_at > NOW())")
|
||||
MEM_EXPIRE_7D=$(DB_QUERY "SELECT count(*) FROM users WHERE is_member AND membership_expires_at IS NOT NULL AND membership_expires_at BETWEEN NOW() AND NOW() + INTERVAL '7 days'")
|
||||
fi
|
||||
[ -z "$PAY_ORDERS_TODAY" ] && PAY_ORDERS_TODAY="N/A"
|
||||
[ -z "$PAY_PAID_TODAY" ] && PAY_PAID_TODAY="N/A"
|
||||
[ -z "$PAY_STUCK" ] && PAY_STUCK="N/A"
|
||||
[ -z "$MEM_ACTIVE" ] && MEM_ACTIVE="N/A"
|
||||
[ -z "$MEM_EXPIRE_7D" ] && MEM_EXPIRE_7D="N/A"
|
||||
|
||||
MAIL=$(cat <<EOF
|
||||
===========================================
|
||||
ASD 服务器健康报告
|
||||
${DATE}
|
||||
主机: ${HOST}
|
||||
===========================================
|
||||
# ---------- 组装 HTML ----------
|
||||
|
||||
🖥 系统状态
|
||||
- 运行时长: ${UPTIME}
|
||||
- 负载: ${LOAD}
|
||||
- 磁盘: ${DISK_USED} / ${DISK_TOTAL} (${DISK_PCT})
|
||||
- 内存: ${MEM_USED} / ${MEM_TOTAL} (${MEM_PCT})
|
||||
- CPU 温度: ${CPU_TEMP}
|
||||
# 行生成器: row <指标> <当前值> <级别> <附加说明?>
|
||||
row() {
|
||||
echo "<tr><td style='padding:6px 10px;border-bottom:1px solid #e2e8f0;color:#475569'>$1</td><td style='padding:6px 10px;border-bottom:1px solid #e2e8f0'>$2</td><td style='padding:6px 10px;border-bottom:1px solid #e2e8f0;text-align:center'>$(badge "$3" "$4")</td></tr>"
|
||||
}
|
||||
section_title() {
|
||||
echo "<tr><td colspan='3' style='padding:10px;background:#f8fafc;font-weight:bold;color:#0f172a;border-bottom:1px solid #e2e8f0'>$1</td></tr>"
|
||||
}
|
||||
|
||||
📦 容器
|
||||
$(echo "$DOCKER_STATUS" | sed 's/^/ /')
|
||||
STYLE="font-family:-apple-system,'PingFang SC','Microsoft YaHei',sans-serif;background:#fff;max-width:640px;margin:0 auto;padding:16px"
|
||||
TH="padding:8px 10px;background:#f1f5f9;color:#64748b;text-align:left;font-size:12px;border-bottom:2px solid #e2e8f0"
|
||||
|
||||
🔐 SSL 证书
|
||||
域名: xmclassmate.top
|
||||
到期: ${SSL_EXPIRY}
|
||||
剩余: ${SSL_DAYS} 天
|
||||
BODY=$(cat <<EOF
|
||||
<html><body style="${STYLE}">
|
||||
<h2 style="color:#0f172a;margin:0 0 4px">ASD 服务器健康报告</h2>
|
||||
<div style="color:#94a3b8;font-size:13px;margin-bottom:16px">${DATE} · ${HOST} · 发送于 $(date "+%H:%M")</div>
|
||||
|
||||
🔄 服务状态
|
||||
- 后端 (green): ${GREEN_STATUS}
|
||||
- 后端 (blue): ${BLUE_STATUS}
|
||||
- nginx: ${NGINX_STATUS}
|
||||
<table style="border-collapse:collapse;width:100%;font-size:13px">
|
||||
<tr><th style="${TH}">指标</th><th style="${TH}">当前值</th><th style="${TH}" width="90">状态</th></tr>
|
||||
|
||||
🛡 安全
|
||||
- SSH 封禁: ${SSH_BANS} 个 IP
|
||||
- nginx 封禁: ${NGINX_BANS} 个 IP
|
||||
- 昨日错误: ${YESTERDAY_ERRORS}
|
||||
- 昨日警告: ${YESTERDAY_WARNS}
|
||||
$(section_title "🖥 系统状态")
|
||||
$(row "运行时长" "$UPTIME" ok "OK")
|
||||
$(row "系统负载" "$LOAD" "$([ "$LOAD1_INT" -ge 4 ] && echo crit || { [ "$LOAD1_INT" -ge 2 ] && echo warn || echo ok; })" "$([ "$LOAD1_INT" -ge 4 ] && echo "超载" || { [ "$LOAD1_INT" -ge 2 ] && echo "偏高" || echo "正常"; })")
|
||||
$(row "磁盘使用" "${DISK_USED} / ${DISK_TOTAL} (${DISK_PCT_STR})" "$(pct_level "$DISK_PCT")" "$(pct_level "$DISK_PCT" | tr 'a-z' 'A-Z')")
|
||||
$(row "内存使用" "${MEM_USED} / ${MEM_TOTAL} (${MEM_PCT}%)" "$(pct_level "$MEM_PCT" 80 90)" "$(pct_level "$MEM_PCT" 80 90 | tr 'a-z' 'A-Z')")
|
||||
$(row "CPU 温度" "${CPU_TEMP}" ok "OK")
|
||||
|
||||
📊 应用概况（昨日）
|
||||
- 日志事件数: ${API_COUNT}
|
||||
- 活跃用户操作: ${ACTIVE_USERS}
|
||||
$(section_title "📦 容器与服务")
|
||||
$(row "Docker 容器" "运行中 $(echo "$DOCKER_STATUS" | grep -c "Up " || echo 0) 个 / 异常 $(echo "$DOCKER_STATUS" | grep -vc "Up " || echo 0) 个" "$([ "$DOCKER_DOWN" -gt 0 ] && echo crit || echo ok)" "$([ "$DOCKER_DOWN" -gt 0 ] && echo "有异常" || echo "正常")")
|
||||
$(row "后端 green" "$GREEN_STATUS" "$([ "$GREEN_STATUS" = "active" ] && echo ok || echo crit)" "$([ "$GREEN_STATUS" = "active" ] && echo "运行中" || echo "异常")")
|
||||
$(row "后端 blue" "$BLUE_STATUS" "$([ "$BLUE_STATUS" = "active" ] && echo ok || echo crit)" "$([ "$BLUE_STATUS" = "active" ] && echo "运行中" || echo "异常")")
|
||||
$(row "nginx" "$NGINX_STATUS" "$([ "$NGINX_STATUS" = "active" ] && echo ok || echo crit)" "$([ "$NGINX_STATUS" = "active" ] && echo "运行中" || echo "异常")")
|
||||
<tr><td colspan="3" style="padding:6px 10px;color:#64748b;font-size:12px;background:#fbfdff">${DOCKER_BODY}</td></tr>
|
||||
|
||||
💾 备份
|
||||
- 最新备份: ${LAST_BACKUP:-无}
|
||||
- 备份时效: ${BACKUP_AGE}
|
||||
$(section_title "🔐 SSL 与安全")
|
||||
$(row "SSL 证书" "${SSL_EXPIRY}（剩余 ${SSL_DAYS} 天）" "$([ "$SSL_DAYS" -lt 7 ] && echo crit || { [ "$SSL_DAYS" -lt 30 ] && echo warn || echo ok; })" "$([ "$SSL_DAYS" -lt 7 ] && echo "即将过期" || { [ "$SSL_DAYS" -lt 30 ] && echo "注意续期" || echo "正常"; })")
|
||||
$(row "Fail2ban SSH" "${SSH_BANS} 个 IP 被封禁" ok "OK")
|
||||
$(row "Fail2ban nginx" "${NGINX_BANS} 个 IP 被封禁" ok "OK")
|
||||
$(row "昨日日志错误" "$YESTERDAY_ERRORS 条" "$([ "$YESTERDAY_ERRORS" -gt 50 ] && echo warn || echo ok)" "$([ "$YESTERDAY_ERRORS" -gt 50 ] && echo "偏多" || echo "正常")")
|
||||
$(row "昨日日志警告" "$YESTERDAY_WARNS 条" "$([ "$YESTERDAY_WARNS" -gt 200 ] && echo warn || echo ok)" "$([ "$YESTERDAY_WARNS" -gt 200 ] && echo "偏多" || echo "正常")")
|
||||
|
||||
---
|
||||
ASD 自动监控 | $(date "+%Y-%m-%d %H:%M")
|
||||
$(section_title "💾 备份")
|
||||
$(row "最新备份" "${LAST_BACKUP:-无}" "$([ -n "$BACKUP_AGE_HOURS" ] && { [ "$BACKUP_AGE_HOURS" -ge 48 ] && echo crit || { [ "$BACKUP_AGE_HOURS" -ge 24 ] && echo warn || echo ok; }; } || echo crit)" "$([ -n "$BACKUP_AGE_HOURS" ] && { [ "$BACKUP_AGE_HOURS" -ge 48 ] && echo "严重过期" || { [ "$BACKUP_AGE_HOURS" -ge 24 ] && echo "⚠ 超24小时" || echo "${BACKUP_AGE_TEXT}"; }; } || echo "未知")")
|
||||
$(row "备份时效" "${BACKUP_AGE_TEXT}" ok "OK")
|
||||
|
||||
$(section_title "📊 应用概况（昨日）")
|
||||
$(row "日志事件数" "$API_COUNT 条" ok "OK")
|
||||
$(row "活跃用户操作" "$ACTIVE_USERS 次" ok "OK")
|
||||
|
||||
$(section_title "💰 支付与会员")
|
||||
$(row "今日新订单" "$PAY_ORDERS_TODAY 笔" ok "OK")
|
||||
$(row "今日支付成功" "$PAY_PAID_TODAY 笔" ok "OK")
|
||||
$(row "卡单（pending>30分钟）" "$PAY_STUCK 笔" "$([ "$PAY_STUCK" -gt 0 ] 2>/dev/null && echo crit || echo ok)" "$([ "$PAY_STUCK" -gt 0 ] 2>/dev/null && echo "需人工处理" || echo "无")")
|
||||
$(row "活跃会员" "$MEM_ACTIVE 人" ok "OK")
|
||||
$(row "7天内到期会员" "$MEM_EXPIRE_7D 人" "$([ "$MEM_EXPIRE_7D" -gt 0 ] 2>/dev/null && echo warn || echo ok)" "$([ "$MEM_EXPIRE_7D" -gt 0 ] 2>/dev/null && echo "可发续费提醒" || echo "无")")
|
||||
$( [ "$DB_OK" = false ] && echo "<tr><td colspan='3' style='padding:6px 10px;color:#dc2626;font-size:12px;background:#fef2f2'>⚠ 数据库不可用，支付/会员统计缺失</td></tr>" )
|
||||
</table>
|
||||
|
||||
<div style="color:#94a3b8;font-size:12px;margin-top:16px">ASD 自动监控 | $(date "+%Y-%m-%d %H:%M") | 异常项请关注 alert.sh 实时告警</div>
|
||||
</body></html>
|
||||
EOF
|
||||
)
|
||||
|
||||
# ---------- 发送邮件 ----------
|
||||
|
||||
echo "$MAIL" | msmtp -a default "$TO"
|
||||
echo "✅ 健康报告已发送到 ${TO}"
|
||||
# ---------- 发送 ----------
|
||||
if mail_send "$SUBJECT" "$BODY"; then
|
||||
echo "✅ 健康报告已发送到 ${MAIL_TO}（$(date '+%H:%M:%S')）"
|
||||
else
|
||||
echo "❌ 健康报告发送失败，详见 ${MAIL_LOG_DIR}/mail.log"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user