208 lines
12 KiB
Bash
208 lines
12 KiB
Bash
#!/bin/bash
|
||
# ===========================================
|
||
# 每日健康报告 — HTML 摘要邮件（含支付/会员统计）
|
||
#
|
||
# 部署：
|
||
# cp scripts/mail-lib.sh scripts/daily-report.sh → 服务器 /root/rust/scripts/
|
||
# chmod +x /root/rust/scripts/daily-report.sh
|
||
# cron（root）:
|
||
# 0 8 * * * /root/rust/scripts/daily-report.sh >> /root/rust/scripts/logs/daily-report.log 2>&1
|
||
#
|
||
# 说明：单项采集失败自动降级为 N/A，不会阻断整封邮件；
|
||
# 发送失败自动重试并记录 /root/rust/scripts/logs/mail.log。
|
||
# ===========================================
|
||
|
||
# 不用 set -e：采集失败不应导致邮件发不出去
|
||
set -u
|
||
|
||
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
|
||
# shellcheck source=mail-lib.sh
|
||
source "${SCRIPT_DIR}/mail-lib.sh"
|
||
|
||
# ---------- 可配置项 ----------
|
||
: "${DB_CONTAINER:=1Panel-postgresql-FtMo}" # PostgreSQL 容器名
|
||
: "${DB_NAME:=milkydata}" # 数据库名
|
||
: "${DB_USER:=milkydata}" # 数据库用户
|
||
: "${BACKUP_DIR:=/root/rust/backups/milkydata}" # 备份目录
|
||
|
||
DATE=$(date "+%Y-%m-%d")
|
||
HOST=$(hostname)
|
||
SUBJECT="[ASD 服务器报告] ${DATE} — ${HOST}"
|
||
|
||
# ---------- 状态徽章 ----------
|
||
# badge <ok|warn|crit> <文本>
|
||
badge() {
|
||
case "$1" in
|
||
ok) echo "<span style='background:#16a34a;color:#fff;padding:1px 8px;border-radius:10px;font-size:12px'>$2</span>" ;;
|
||
warn) echo "<span style='background:#d97706;color:#fff;padding:1px 8px;border-radius:10px;font-size:12px'>$2</span>" ;;
|
||
crit) echo "<span style='background:#dc2626;color:#fff;padding:1px 8px;border-radius:10px;font-size:12px'>$2</span>" ;;
|
||
*) echo "<span style='background:#64748b;color:#fff;padding:1px 8px;border-radius:10px;font-size:12px'>$2</span>" ;;
|
||
esac
|
||
}
|
||
|
||
# 根据磁盘/内存百分比判定级别（参数: 数值，返回 ok/warn/crit/unknown）
|
||
pct_level() {
|
||
local v="$1" w="${2:-80}" c="${3:-90}"
|
||
case "$v" in
|
||
''|*[!0-9]*) echo "unknown"; return ;;
|
||
esac
|
||
if [ "$v" -ge "$c" ]; then echo "crit"
|
||
elif [ "$v" -ge "$w" ]; then echo "warn"
|
||
else echo "ok"; fi
|
||
}
|
||
|
||
# ---------- 系统状态 ----------
|
||
|
||
UPTIME=$(uptime -p | sed 's/up //')
|
||
LOAD=$(uptime | awk -F'load average:' '{print $2}' | xargs)
|
||
LOAD1=${LOAD%%,*} # 1 分钟负载（可能是小数，取整数部分比较）
|
||
LOAD1_INT=${LOAD1%%.*}
|
||
DISK_USED=$(df -h / | awk 'NR==2{print $3}')
|
||
DISK_TOTAL=$(df -h / | awk 'NR==2{print $2}')
|
||
DISK_PCT_STR=$(df -h / | awk 'NR==2{print $5}')
|
||
DISK_PCT=${DISK_PCT_STR%\%}
|
||
MEM_USED=$(free -h | awk '/Mem/{print $3}')
|
||
MEM_TOTAL=$(free -h | awk '/Mem/{print $2}')
|
||
MEM_PCT=$(free | awk '/Mem/{printf "%.0f", $3/$2 * 100}')
|
||
CPU_TEMP=$(cat /sys/class/thermal/thermal_zone0/temp 2>/dev/null | awk '{printf "%.1f°C", $1/1000}' || echo "N/A")
|
||
|
||
# ---------- Docker ----------
|
||
DOCKER_STATUS=$(docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null || echo "")
|
||
DOCKER_DOWN=$(echo "$DOCKER_STATUS" | grep -vc "Up " || true)
|
||
if [ -z "$DOCKER_STATUS" ]; then
|
||
DOCKER_BODY="<span style='color:#dc2626'>Docker 不可用或无运行容器</span>"
|
||
elif [ "$DOCKER_DOWN" -gt 0 ]; then
|
||
DOCKER_BODY="$(echo "$DOCKER_STATUS" | awk '{print $1": "$2}' | sed 's/^/ /' | sed 's/&/\&/g;s/</\</g;s/>/\>/g' | tr '\n' '<br>')"
|
||
else
|
||
DOCKER_BODY="$(echo "$DOCKER_STATUS" | awk '{print $1" "$2}' | sed 's/&/\&/g;s/</\</g;s/>/\>/g' | tr '\n' '<br>')"
|
||
fi
|
||
|
||
# ---------- SSL 证书 ----------
|
||
SSL_EXPIRY=$(echo "" | openssl s_client -connect xmclassmate.top:443 -servername xmclassmate.top 2>&1 | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2 || echo "检测失败")
|
||
if [ "$SSL_EXPIRY" != "检测失败" ]; then
|
||
SSL_DAYS=$(( ($(date -d "$SSL_EXPIRY" +%s 2>/dev/null) - $(date +%s)) / 86400 ))
|
||
else
|
||
SSL_DAYS=9999
|
||
fi
|
||
|
||
# ---------- 服务状态 ----------
|
||
GREEN_STATUS=$(systemctl is-active rust-backend-green.service 2>/dev/null || echo "unknown")
|
||
BLUE_STATUS=$(systemctl is-active rust-backend-blue.service 2>/dev/null || echo "unknown")
|
||
NGINX_STATUS=$(systemctl is-active nginx 2>/dev/null || echo "unknown")
|
||
|
||
# ---------- 备份 ----------
|
||
LAST_BACKUP=$(ls -1t "$BACKUP_DIR/" 2>/dev/null | head -1)
|
||
BACKUP_AGE_HOURS=""
|
||
BACKUP_AGE_TEXT="未知"
|
||
if [ -n "$LAST_BACKUP" ]; then
|
||
BACKUP_TIME=$(stat -c %Y "$BACKUP_DIR/$LAST_BACKUP" 2>/dev/null)
|
||
NOW=$(date +%s)
|
||
BACKUP_AGE_HOURS=$(( (NOW - BACKUP_TIME) / 3600 ))
|
||
BACKUP_AGE_TEXT="${BACKUP_AGE_HOURS} 小时前"
|
||
fi
|
||
|
||
# ---------- 安全 ----------
|
||
SSH_BANS=$(sudo fail2ban-client status sshd 2>/dev/null | grep "Currently banned" | awk '{print $NF}' || echo "0")
|
||
NGINX_BANS=$(sudo fail2ban-client status nginx-botsearch 2>/dev/null | grep "Currently banned" | awk '{print $NF}' || echo "0")
|
||
YESTERDAY_ERRORS=$(journalctl -u rust-backend-green.service --since "1 day ago" --priority err --no-pager 2>/dev/null | wc -l)
|
||
YESTERDAY_WARNS=$(journalctl -u rust-backend-green.service --since "1 day ago" --priority warning --no-pager 2>/dev/null | wc -l)
|
||
|
||
# ---------- 应用概况（昨日） ----------
|
||
ACTIVE_USERS=$(journalctl -u rust-backend-green.service --since "1 day ago" --no-pager 2>/dev/null | grep -c "user_id=\|登录成功\|上传成功" || echo "0")
|
||
API_COUNT=$(journalctl -u rust-backend-green.service --since "1 day ago" --no-pager 2>/dev/null | grep -c "INFO\|WARN\|ERROR" || echo "0")
|
||
|
||
# ---------- 支付与会员（数据库统计，不可用则 N/A） ----------
|
||
# 服务器上通过 docker exec 访问 PostgreSQL（与 check-members.sh 同模式）
|
||
DB_QUERY() {
|
||
docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c "$1" 2>/dev/null || echo ""
|
||
}
|
||
DB_OK=false
|
||
if [ -n "$(DB_QUERY 'SELECT 1')" ]; then DB_OK=true; fi
|
||
|
||
PAY_ORDERS_TODAY="N/A"; PAY_PAID_TODAY="N/A"; PAY_STUCK="N/A"
|
||
MEM_ACTIVE="N/A"; MEM_EXPIRE_7D="N/A"
|
||
if [ "$DB_OK" = true ]; then
|
||
PAY_ORDERS_TODAY=$(DB_QUERY "SELECT count(*) FROM payment_orders WHERE created_at::date = CURRENT_DATE")
|
||
PAY_PAID_TODAY=$(DB_QUERY "SELECT count(*) FROM payment_orders WHERE status='paid' AND paid_at::date = CURRENT_DATE")
|
||
PAY_STUCK=$(DB_QUERY "SELECT count(*) FROM payment_orders WHERE status='pending' AND created_at < NOW() - INTERVAL '30 minutes'")
|
||
MEM_ACTIVE=$(DB_QUERY "SELECT count(*) FROM users WHERE is_member AND (membership_expires_at IS NULL OR membership_expires_at > NOW())")
|
||
MEM_EXPIRE_7D=$(DB_QUERY "SELECT count(*) FROM users WHERE is_member AND membership_expires_at IS NOT NULL AND membership_expires_at BETWEEN NOW() AND NOW() + INTERVAL '7 days'")
|
||
fi
|
||
[ -z "$PAY_ORDERS_TODAY" ] && PAY_ORDERS_TODAY="N/A"
|
||
[ -z "$PAY_PAID_TODAY" ] && PAY_PAID_TODAY="N/A"
|
||
[ -z "$PAY_STUCK" ] && PAY_STUCK="N/A"
|
||
[ -z "$MEM_ACTIVE" ] && MEM_ACTIVE="N/A"
|
||
[ -z "$MEM_EXPIRE_7D" ] && MEM_EXPIRE_7D="N/A"
|
||
|
||
# ---------- 组装 HTML ----------
|
||
|
||
# 行生成器: row <指标> <当前值> <级别> <附加说明?>
|
||
row() {
|
||
echo "<tr><td style='padding:6px 10px;border-bottom:1px solid #e2e8f0;color:#475569'>$1</td><td style='padding:6px 10px;border-bottom:1px solid #e2e8f0'>$2</td><td style='padding:6px 10px;border-bottom:1px solid #e2e8f0;text-align:center'>$(badge "$3" "$4")</td></tr>"
|
||
}
|
||
section_title() {
|
||
echo "<tr><td colspan='3' style='padding:10px;background:#f8fafc;font-weight:bold;color:#0f172a;border-bottom:1px solid #e2e8f0'>$1</td></tr>"
|
||
}
|
||
|
||
STYLE="font-family:-apple-system,'PingFang SC','Microsoft YaHei',sans-serif;background:#fff;max-width:640px;margin:0 auto;padding:16px"
|
||
TH="padding:8px 10px;background:#f1f5f9;color:#64748b;text-align:left;font-size:12px;border-bottom:2px solid #e2e8f0"
|
||
|
||
BODY=$(cat <<EOF
|
||
<html><body style="${STYLE}">
|
||
<h2 style="color:#0f172a;margin:0 0 4px">ASD 服务器健康报告</h2>
|
||
<div style="color:#94a3b8;font-size:13px;margin-bottom:16px">${DATE} · ${HOST} · 发送于 $(date "+%H:%M")</div>
|
||
|
||
<table style="border-collapse:collapse;width:100%;font-size:13px">
|
||
<tr><th style="${TH}">指标</th><th style="${TH}">当前值</th><th style="${TH}" width="90">状态</th></tr>
|
||
|
||
$(section_title "🖥 系统状态")
|
||
$(row "运行时长" "$UPTIME" ok "OK")
|
||
$(row "系统负载" "$LOAD" "$([ "$LOAD1_INT" -ge 4 ] && echo crit || { [ "$LOAD1_INT" -ge 2 ] && echo warn || echo ok; })" "$([ "$LOAD1_INT" -ge 4 ] && echo "超载" || { [ "$LOAD1_INT" -ge 2 ] && echo "偏高" || echo "正常"; })")
|
||
$(row "磁盘使用" "${DISK_USED} / ${DISK_TOTAL} (${DISK_PCT_STR})" "$(pct_level "$DISK_PCT")" "$(pct_level "$DISK_PCT" | tr 'a-z' 'A-Z')")
|
||
$(row "内存使用" "${MEM_USED} / ${MEM_TOTAL} (${MEM_PCT}%)" "$(pct_level "$MEM_PCT" 80 90)" "$(pct_level "$MEM_PCT" 80 90 | tr 'a-z' 'A-Z')")
|
||
$(row "CPU 温度" "${CPU_TEMP}" ok "OK")
|
||
|
||
$(section_title "📦 容器与服务")
|
||
$(row "Docker 容器" "运行中 $(echo "$DOCKER_STATUS" | grep -c "Up " || echo 0) 个 / 异常 $(echo "$DOCKER_STATUS" | grep -vc "Up " || echo 0) 个" "$([ "$DOCKER_DOWN" -gt 0 ] && echo crit || echo ok)" "$([ "$DOCKER_DOWN" -gt 0 ] && echo "有异常" || echo "正常")")
|
||
$(row "后端 green" "$GREEN_STATUS" "$([ "$GREEN_STATUS" = "active" ] && echo ok || echo crit)" "$([ "$GREEN_STATUS" = "active" ] && echo "运行中" || echo "异常")")
|
||
$(row "后端 blue" "$BLUE_STATUS" "$([ "$BLUE_STATUS" = "active" ] && echo ok || echo crit)" "$([ "$BLUE_STATUS" = "active" ] && echo "运行中" || echo "异常")")
|
||
$(row "nginx" "$NGINX_STATUS" "$([ "$NGINX_STATUS" = "active" ] && echo ok || echo crit)" "$([ "$NGINX_STATUS" = "active" ] && echo "运行中" || echo "异常")")
|
||
<tr><td colspan="3" style="padding:6px 10px;color:#64748b;font-size:12px;background:#fbfdff">${DOCKER_BODY}</td></tr>
|
||
|
||
$(section_title "🔐 SSL 与安全")
|
||
$(row "SSL 证书" "${SSL_EXPIRY}（剩余 ${SSL_DAYS} 天）" "$([ "$SSL_DAYS" -lt 7 ] && echo crit || { [ "$SSL_DAYS" -lt 30 ] && echo warn || echo ok; })" "$([ "$SSL_DAYS" -lt 7 ] && echo "即将过期" || { [ "$SSL_DAYS" -lt 30 ] && echo "注意续期" || echo "正常"; })")
|
||
$(row "Fail2ban SSH" "${SSH_BANS} 个 IP 被封禁" ok "OK")
|
||
$(row "Fail2ban nginx" "${NGINX_BANS} 个 IP 被封禁" ok "OK")
|
||
$(row "昨日日志错误" "$YESTERDAY_ERRORS 条" "$([ "$YESTERDAY_ERRORS" -gt 50 ] && echo warn || echo ok)" "$([ "$YESTERDAY_ERRORS" -gt 50 ] && echo "偏多" || echo "正常")")
|
||
$(row "昨日日志警告" "$YESTERDAY_WARNS 条" "$([ "$YESTERDAY_WARNS" -gt 200 ] && echo warn || echo ok)" "$([ "$YESTERDAY_WARNS" -gt 200 ] && echo "偏多" || echo "正常")")
|
||
|
||
$(section_title "💾 备份")
|
||
$(row "最新备份" "${LAST_BACKUP:-无}" "$([ -n "$BACKUP_AGE_HOURS" ] && { [ "$BACKUP_AGE_HOURS" -ge 48 ] && echo crit || { [ "$BACKUP_AGE_HOURS" -ge 24 ] && echo warn || echo ok; }; } || echo crit)" "$([ -n "$BACKUP_AGE_HOURS" ] && { [ "$BACKUP_AGE_HOURS" -ge 48 ] && echo "严重过期" || { [ "$BACKUP_AGE_HOURS" -ge 24 ] && echo "⚠ 超24小时" || echo "${BACKUP_AGE_TEXT}"; }; } || echo "未知")")
|
||
$(row "备份时效" "${BACKUP_AGE_TEXT}" ok "OK")
|
||
|
||
$(section_title "📊 应用概况（昨日）")
|
||
$(row "日志事件数" "$API_COUNT 条" ok "OK")
|
||
$(row "活跃用户操作" "$ACTIVE_USERS 次" ok "OK")
|
||
|
||
$(section_title "💰 支付与会员")
|
||
$(row "今日新订单" "$PAY_ORDERS_TODAY 笔" ok "OK")
|
||
$(row "今日支付成功" "$PAY_PAID_TODAY 笔" ok "OK")
|
||
$(row "卡单（pending>30分钟）" "$PAY_STUCK 笔" "$([ "$PAY_STUCK" -gt 0 ] 2>/dev/null && echo crit || echo ok)" "$([ "$PAY_STUCK" -gt 0 ] 2>/dev/null && echo "需人工处理" || echo "无")")
|
||
$(row "活跃会员" "$MEM_ACTIVE 人" ok "OK")
|
||
$(row "7天内到期会员" "$MEM_EXPIRE_7D 人" "$([ "$MEM_EXPIRE_7D" -gt 0 ] 2>/dev/null && echo warn || echo ok)" "$([ "$MEM_EXPIRE_7D" -gt 0 ] 2>/dev/null && echo "可发续费提醒" || echo "无")")
|
||
$( [ "$DB_OK" = false ] && echo "<tr><td colspan='3' style='padding:6px 10px;color:#dc2626;font-size:12px;background:#fef2f2'>⚠ 数据库不可用，支付/会员统计缺失</td></tr>" )
|
||
</table>
|
||
|
||
<div style="color:#94a3b8;font-size:12px;margin-top:16px">ASD 自动监控 | $(date "+%Y-%m-%d %H:%M") | 异常项请关注 alert.sh 实时告警</div>
|
||
</body></html>
|
||
EOF
|
||
)
|
||
|
||
# ---------- 发送 ----------
|
||
if mail_send "$SUBJECT" "$BODY"; then
|
||
echo "✅ 健康报告已发送到 ${MAIL_TO}（$(date '+%H:%M:%S')）"
|
||
else
|
||
echo "❌ 健康报告发送失败，详见 ${MAIL_LOG_DIR}/mail.log"
|
||
exit 1
|
||
fi
|