diff --git a/docs/OPS-ALERT.md b/docs/OPS-ALERT.md new file mode 100644 index 0000000..6599486 --- /dev/null +++ b/docs/OPS-ALERT.md @@ -0,0 +1,84 @@ +# 服务器邮件告警与每日报告 — 部署指南 + + + +## 概述 + +两封自动邮件，均通过 `msmtp` → 163 SMTP 发送（复用现有 `~/.msmtprc` 配置）： + +| 脚本 | 频率（cron） | 作用 | +| --- | --- | --- | +| `daily-report.sh` | 每日 08:00 | 服务器健康报告（HTML）：系统/容器/SSL/服务/安全/备份 + **支付与会员统计** | +| `alert.sh` | 每 5 分钟 | **实时异常告警**：正常时静默，异常才发邮件（带防刷屏状态机） | + +## 部署步骤 + +```bash +# 1. 上传三个文件到服务器（root 用户） +scp scripts/mail-lib.sh scripts/daily-report.sh scripts/alert.sh root@<服务器>:/root/rust/scripts/ +ssh root@<服务器> 'chmod +x /root/rust/scripts/*.sh && mkdir -p /root/rust/scripts/logs' + +# 2. 配置 cron（root） +ssh root@<服务器> 'crontab -e' +# 加入以下两行： +0 8 * * * /root/rust/scripts/daily-report.sh >> /root/rust/scripts/logs/daily-report.log 2>&1 +*/5 * * * * /root/rust/scripts/alert.sh >> /root/rust/scripts/logs/alert.log 2>&1 +``` + +## 验证 + +```bash +# 干跑告警（预览当前异常与将发送的邮件，不真正发送） +/root/rust/scripts/alert.sh --dry-run + +# 手动发一封健康报告测试 msmtp 链路 +/root/rust/scripts/daily-report.sh + +# 查看邮件日志 +tail -50 /root/rust/scripts/logs/mail.log +``` + +## alert.sh 检测项与阈值 + +| 检测项 | 警告 | 严重 | +| --- | --- | --- | +| 磁盘使用（/） | > 80% | > 90% | +| 内存使用 | > 80% | > 90% | +| 备份时效 | > 24h | > 48h（或无备份） | +| 服务状态（green/blue/nginx） | — | 非 active | +| Docker 容器 | — | 存在非 Up 容器 / docker 不可用 | +| SSL 证书剩余 | < 30 天 | < 7 天 | +| 支付卡单（pending > 30 分钟） | ≥ 1 笔（附订单号） | — | +| 数据库（SELECT 1） | — | 不可用 | + +### 防刷屏机制 + +状态文件 `.alert-state`（默认 `/root/rust/scripts/.alert-state`）记录每个检测项最近告警级别： + +- 级别**上升** → 立即发邮件 +- 级别**不变** → 冷却 `ALERT_COOLDOWN_HOURS`（默认 6 小时）后才再次提醒 +- **恢复**正常 → 静默清除状态，不发邮件 + +## 可配置项（环境变量） + +| 变量 | 默认值 | 说明 | +| --- | --- | --- | +| `MAIL_TO` | `m943790568@163.com` | 收件人，逗号分隔多地址 | +| `MAIL_FROM` | `ASD 监控 ` | 发件人显示名 | +| `MAIL_ACCOUNT` | `default` | msmtp 账户名 | +| `MAIL_LOG_DIR` | `/root/rust/scripts/logs` | 日志目录（mail.log / alert.log） | +| `MAIL_RETRY` | `3` | 发送失败重试次数 | +| `DB_CONTAINER` | `1Panel-postgresql-FtMo` | PostgreSQL 容器名（1Panel 安装） | +| `BACKUP_DIR` | `/root/rust/backups/milkydata` | 备份目录 | +| `STUCK_ORDER_MINUTES` | `30` | pending 订单视为卡单的时长 | +| `ALERT_STATE_FILE` | `/root/rust/scripts/.alert-state` | 状态文件路径 | +| `ALERT_COOLDOWN_HOURS` | `6` | 同级别再次提醒的冷却时间 | + +## 注意事项 + +- 脚本在**服务器本机**运行（cron 以 root 执行），数据库通过 `docker exec <容器> psql` 访问， + 容器名与 `check-members.sh` / `send-notification.sh` 保持一致；如容器名不同，设置 `DB_CONTAINER` 覆盖。 +- 邮件为 HTML + UTF-8，无需额外编码配置；msmtp 发送失败自动重试 3 次并记录日志。 +- 单次采集失败（如 journalctl 无权限）不会阻断整封邮件，对应项显示 N/A。 +- `alert.sh` 的卡单检测可直接发现「支付卡在确认页面」类问题（pending 超 30 分钟即告警， + 邮件中附订单号与创建时间，便于对照支付宝账单处理）。 diff --git a/scripts/alert.sh b/scripts/alert.sh new file mode 100644 index 0000000..b59fb08 --- /dev/null +++ b/scripts/alert.sh @@ -0,0 +1,257 @@ +#!/bin/bash +# =========================================== +# alert.sh — 服务器异常实时告警（正常时静默，异常才发邮件） +# +# 检测项： +# 磁盘使用 >80% 警告 / >90% 严重 +# 内存使用 >80% 警告 / >90% 严重 +# 备份时效 >24h 警告 / >48h 严重（含备份缺失） +# 服务状态 green/blue/nginx 非 active → 严重 +# Docker 容器非 Up → 严重 +# SSL 剩余 <30 天警告 / <7 天严重 +# 支付卡单 pending 且创建超过 30 分钟 → 警告（列出订单号） +# 数据库 不可用 → 严重（支付/会员功能受影响） +# +# 防刷屏状态机（.alert-state）： +# 级别上升 → 立即告警；级别不变 → 超过 ALERT_COOLDOWN_HOURS 再次提醒； +# 恢复正常 → 清除状态，不打扰。 +# +# 部署： +# cp scripts/mail-lib.sh scripts/alert.sh → 服务器 /root/rust/scripts/ +# cron（root）: +# */5 * * * * /root/rust/scripts/alert.sh >> /root/rust/scripts/logs/alert.log 2>&1 +# +# 用法：/root/rust/scripts/alert.sh [--dry-run] +# =========================================== + +set -u + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=mail-lib.sh +source "${SCRIPT_DIR}/mail-lib.sh" + +# ---------- 可配置项（环境变量可覆盖） ---------- +: "${ALERT_STATE_FILE:=/root/rust/scripts/.alert-state}" +: "${ALERT_COOLDOWN_HOURS:=6}" # 同一级别再次提醒的冷却时间 +: "${DB_CONTAINER:=1Panel-postgresql-FtMo}" +: "${DB_NAME:=milkydata}" +: "${DB_USER:=milkydata}" +: "${STUCK_ORDER_MINUTES:=30}" # pending 超过该时长视为卡单 +: "${BACKUP_DIR:=/root/rust/backups/milkydata}" # 备份目录 + +DRY_RUN=false +[ "${1:-}" = "--dry-run" ] && DRY_RUN=true + +# 日志 +alert_log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "${MAIL_LOG_DIR}/alert.log" 2>/dev/null || true; } +[ "$DRY_RUN" = true ] && alert_log() { echo "[DRY-RUN] $*"; } + +# ---------- 检测项 ---------- +# 每项输出: 级别(0/1/2)|键名|标签|当前值|详情 +# 级别: 0=正常 1=警告 2=严重 + +check_disk() { + local pct; pct=$(df -P / | awk 'NR==2{print $5}' | tr -d '%') + pct=${pct:-0} + local used total; used=$(df -h / | awk 'NR==2{print $3}'); total=$(df -h / | awk 'NR==2{print $2}') + local lvl=0 + [ "$pct" -ge 90 ] && lvl=2 + [ "$pct" -ge 80 ] && [ "$lvl" -eq 0 ] && lvl=1 + echo "${lvl}|disk|磁盘使用|${used} / ${total} (${pct}%)|根分区已用 ${pct}%" +} + +check_mem() { + local pct; pct=$(free | awk '/Mem/{printf "%.0f", $3/$2 * 100}') + pct=${pct:-0} + local used total; used=$(free -h | awk '/Mem/{print $3}'); total=$(free -h | awk '/Mem/{print $2}') + local lvl=0 + [ "$pct" -ge 90 ] && lvl=2 + [ "$pct" -ge 80 ] && [ "$lvl" -eq 0 ] && lvl=1 + echo "${lvl}|mem|内存使用|${used} / ${total} (${pct}%)|内存已用 ${pct}%" +} + +check_backup() { + local dir="$BACKUP_DIR" + local last; last=$(ls -1t "$dir" 2>/dev/null | head -1) + if [ -z "$last" ]; then + echo "2|backup|备份|无备份文件|目录 ${dir} 无任何备份" + return + fi + local age; age=$(( ($(date +%s) - $(stat -c %Y "$dir/$last" 2>/dev/null || echo 0)) / 3600 )) + local lvl=0 + [ "$age" -ge 48 ] && lvl=2 + [ "$age" -ge 24 ] && [ "$lvl" -eq 0 ] && lvl=1 + echo "${lvl}|backup|备份时效|${age} 小时前（${last}）|最新备份距今 ${age} 小时" +} + +check_services() { + local bad="" + for svc in rust-backend-green rust-backend-blue nginx; do + local st; st=$(systemctl is-active "$svc" 2>/dev/null || echo "unknown") + [ "$st" = "active" ] || bad="${bad}${svc}=${st} " + done + if [ -n "$bad" ]; then + echo "2|services|服务状态|异常服务: ${bad}|systemd 服务未运行" + else + echo "0|services|服务状态|全部 active|green/blue/nginx 运行正常" + fi +} + +check_docker() { + local out; out=$(docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null) + if [ -z "$out" ]; then + if docker info >/dev/null 2>&1; then + echo "0|docker|Docker 容器|无运行容器|当前无容器在运行" + else + echo "2|docker|Docker|不可用|docker 命令执行失败" + fi + return + fi + local bad; bad=$(echo "$out" | grep -v " Up " || true) + if [ -n "$bad" ]; then + echo "2|docker|Docker 容器|$(echo "$out" | grep -c " Up ") 个正常|异常容器: $(echo "$bad" | awk '{print $1}' | tr '\n' ' ')" + else + echo "0|docker|Docker 容器|$(echo "$out" | grep -c " Up ") 个运行中|全部正常" + fi +} + +check_ssl() { + local expiry; expiry=$(echo "" | openssl s_client -connect xmclassmate.top:443 -servername xmclassmate.top 2>&1 | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) + if [ -z "$expiry" ]; then + echo "1|ssl|SSL 证书|检测失败|无法获取证书到期时间" + return + fi + local days; days=$(( ($(date -d "$expiry" +%s 2>/dev/null || echo 0) - $(date +%s)) / 86400 )) + local lvl=0 + [ "$days" -lt 7 ] && lvl=2 + [ "$days" -lt 30 ] && [ "$lvl" -eq 0 ] && lvl=1 + echo "${lvl}|ssl|SSL 证书|剩余 ${days} 天（${expiry}）|证书将于 ${expiry} 到期" +} + +# 支付卡单检测（pending 超过 STUCK_ORDER_MINUTES） +check_stuck_orders() { + local count orders + count=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c \ + "SELECT count(*) FROM payment_orders WHERE status='pending' AND created_at < NOW() - INTERVAL '${STUCK_ORDER_MINUTES} minutes'" 2>/dev/null) + if [ -z "$count" ]; then + return # 数据库不可用由 check_db 负责 + fi + if [ "$count" -gt 0 ]; then + orders=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c \ + "SELECT order_no || ' (用户' || user_id || ', ' || to_char(created_at, 'MM-DD HH24:MI') || ')' FROM payment_orders WHERE status='pending' AND created_at < NOW() - INTERVAL '${STUCK_ORDER_MINUTES} minutes' ORDER BY created_at" 2>/dev/null | tr '\n' '; ') + echo "1|stuck_orders|支付卡单|${count} 笔 pending 超 ${STUCK_ORDER_MINUTES} 分钟|${orders}" + else + echo "0|stuck_orders|支付卡单|无|无卡单" + fi +} + +check_db() { + local ok; ok=$(docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c "SELECT 1" 2>/dev/null) + if [ -z "$ok" ]; then + echo "2|db|数据库|不可用|docker exec psql 失败（容器 ${DB_CONTAINER}）" + else + echo "0|db|数据库|正常|SELECT 1 成功" + fi +} + +# ---------- 状态文件管理 ---------- +# 格式: key lvl last_alert_epoch +load_state() { [ -f "$ALERT_STATE_FILE" ] && cat "$ALERT_STATE_FILE" || true; } + +save_state() { + local tmp="${ALERT_STATE_FILE}.tmp" + { for k in "${!NEW_STATE[@]}"; do echo "$k ${NEW_STATE[$k]}"; done; } > "$tmp" + mv "$tmp" "$ALERT_STATE_FILE" 2>/dev/null || true +} + +# 读取状态：$1=key, $2=字段序号（1=级别 2=时间）；状态行格式为 "key lvl time" +get_state() { + local line; line=$(load_state | awk -v k="$1" '$1==k') + [ -n "$line" ] || { echo "-1"; return; } + echo "$line" | awk -v f="$2" '{print $(f+1)}' +} + +# ---------- 主流程 ---------- +NOW=$(date +%s) +COOLDOWN_SEC=$((ALERT_COOLDOWN_HOURS * 3600)) + +# 收集检测结果（格式: level|key|label|value|detail） +RESULTS="" +for check in check_disk check_mem check_backup check_services check_docker check_ssl check_db check_stuck_orders; do + r=$($check 2>/dev/null) + [ -n "$r" ] && RESULTS="${RESULTS}${r}\n" +done + +declare -A NEW_STATE +ALERTS="" # 需要发邮件的条目 +WARN_COUNT=0 +CRIT_COUNT=0 + +while IFS='|' read -r lvl key label value detail; do + [ -z "$key" ] && continue + [ "$lvl" -eq 0 ] && continue # 正常项不处理（下面统一清除状态） + + old_lvl=$(get_state "$key" 1) + old_time=$(get_state "$key" 2) + [ "$old_time" = "-1" ] && old_time=0 + + should_alert=false + if [ "$lvl" -gt "$old_lvl" ]; then + should_alert=true # 级别上升 → 立即告警 + elif [ "$lvl" -eq "$old_lvl" ] && [ $((NOW - old_time)) -ge "$COOLDOWN_SEC" ]; then + should_alert=true # 同级别冷却期后再次提醒 + fi + + NEW_STATE[$key]="${lvl} ${NOW}" + if [ "$should_alert" = true ]; then + [ "$lvl" -eq 2 ] && CRIT_COUNT=$((CRIT_COUNT + 1)) + [ "$lvl" -eq 1 ] && WARN_COUNT=$((WARN_COUNT + 1)) + ALERTS="${ALERTS}$( [ "$lvl" -eq 2 ] && echo "严重" || echo "警告" )${label}${value}${detail}\n" + alert_log "告警 [${key}] 级别=${lvl}: ${label} ${value} — ${detail}" + fi +done < <(printf "%b" "$RESULTS") + +# 恢复检测：状态文件中存在但本次为 0/未出现的项 → 清除 +while read -r key old_lvl _; do + [ -z "$key" ] && continue + cur_lvl=$(printf "%b" "$RESULTS" | awk -F'|' -v k="$key" '$2==k{print $1; exit}') + if [ -z "$cur_lvl" ] || [ "$cur_lvl" -eq 0 ]; then + unset 'NEW_STATE[$key]' 2>/dev/null || true + alert_log "恢复: ${key} 已恢复正常，状态已清除" + fi +done < <(load_state) + +save_state + +# ---------- 发送告警邮件 ---------- +if [ -n "$ALERTS" ]; then + if [ "$CRIT_COUNT" -gt 0 ]; then + SUBJECT="[ASD 告警] ${CRIT_COUNT} 项严重 / ${WARN_COUNT} 项警告 — $(date '+%m-%d %H:%M')" + else + SUBJECT="[ASD 告警] ${WARN_COUNT} 项警告 — $(date '+%m-%d %H:%M')" + fi + BODY=$(cat < +

⚠️ ASD 服务器异常告警

+
$(date '+%Y-%m-%d %H:%M:%S') · $(hostname)
+ + +$(printf "%b" "$ALERTS") +
级别项目当前值说明
+
下次同类提醒将在 ${ALERT_COOLDOWN_HOURS} 小时后，恢复正常后不再打扰。
+ +EOF +) + if [ "$DRY_RUN" = true ]; then + echo "---- 将发送告警邮件 ----"; echo "主题: ${SUBJECT}"; echo "$BODY" | head -30 + else + if mail_send "$SUBJECT" "$BODY"; then + alert_log "告警邮件已发送: ${SUBJECT}" + else + alert_log "告警邮件发送失败: ${SUBJECT}" + fi + fi +else + alert_log "本次检查无异常（$(date '+%H:%M:%S')）" + [ "$DRY_RUN" = true ] && echo "当前无异常，无需告警。" +fi diff --git a/scripts/daily-report.sh b/scripts/daily-report.sh index 2fcf78d..1390e2d 100644 --- a/scripts/daily-report.sh +++ b/scripts/daily-report.sh @@ -1,116 +1,207 @@ #!/bin/bash -# 每日健康报告 — 发送格式化摘要邮件到管理员 -# 部署：cp 到服务器 /root/rust/scripts/daily-report.sh，cron 每日执行 +# =========================================== +# 每日健康报告 — HTML 摘要邮件（含支付/会员统计） +# +# 部署： +# cp scripts/mail-lib.sh scripts/daily-report.sh → 服务器 /root/rust/scripts/ +# chmod +x /root/rust/scripts/daily-report.sh +# cron（root）: +# 0 8 * * * /root/rust/scripts/daily-report.sh >> /root/rust/scripts/logs/daily-report.log 2>&1 +# +# 说明：单项采集失败自动降级为 N/A，不会阻断整封邮件； +# 发送失败自动重试并记录 /root/rust/scripts/logs/mail.log。 +# =========================================== -set -e +# 不用 set -e：采集失败不应导致邮件发不出去 +set -u + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=mail-lib.sh +source "${SCRIPT_DIR}/mail-lib.sh" + +# ---------- 可配置项 ---------- +: "${DB_CONTAINER:=1Panel-postgresql-FtMo}" # PostgreSQL 容器名 +: "${DB_NAME:=milkydata}" # 数据库名 +: "${DB_USER:=milkydata}" # 数据库用户 +: "${BACKUP_DIR:=/root/rust/backups/milkydata}" # 备份目录 -TO="m943790568@163.com" DATE=$(date "+%Y-%m-%d") HOST=$(hostname) +SUBJECT="[ASD 服务器报告] ${DATE} — ${HOST}" -# ---------- 采集数据 ---------- +# ---------- 状态徽章 ---------- +# badge <文本> +badge() { + case "$1" in + ok) echo "$2" ;; + warn) echo "$2" ;; + crit) echo "$2" ;; + *) echo "$2" ;; + esac +} + +# 根据磁盘/内存百分比判定级别（参数: 数值，返回 ok/warn/crit/unknown） +pct_level() { + local v="$1" w="${2:-80}" c="${3:-90}" + case "$v" in + ''|*[!0-9]*) echo "unknown"; return ;; + esac + if [ "$v" -ge "$c" ]; then echo "crit" + elif [ "$v" -ge "$w" ]; then echo "warn" + else echo "ok"; fi +} + +# ---------- 系统状态 ---------- UPTIME=$(uptime -p | sed 's/up //') LOAD=$(uptime | awk -F'load average:' '{print $2}' | xargs) +LOAD1=${LOAD%%,*} # 1 分钟负载（可能是小数，取整数部分比较） +LOAD1_INT=${LOAD1%%.*} DISK_USED=$(df -h / | awk 'NR==2{print $3}') DISK_TOTAL=$(df -h / | awk 'NR==2{print $2}') -DISK_PCT=$(df -h / | awk 'NR==2{print $5}') +DISK_PCT_STR=$(df -h / | awk 'NR==2{print $5}') +DISK_PCT=${DISK_PCT_STR%\%} MEM_USED=$(free -h | awk '/Mem/{print $3}') MEM_TOTAL=$(free -h | awk '/Mem/{print $2}') MEM_PCT=$(free | awk '/Mem/{printf "%.0f", $3/$2 * 100}') CPU_TEMP=$(cat /sys/class/thermal/thermal_zone0/temp 2>/dev/null | awk '{printf "%.1f°C", $1/1000}' || echo "N/A") -# Docker -DOCKER_STATUS=$(docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null || echo "Docker 不可用") +# ---------- Docker ---------- +DOCKER_STATUS=$(docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null || echo "") +DOCKER_DOWN=$(echo "$DOCKER_STATUS" | grep -vc "Up " || true) +if [ -z "$DOCKER_STATUS" ]; then + DOCKER_BODY="Docker 不可用或无运行容器" +elif [ "$DOCKER_DOWN" -gt 0 ]; then + DOCKER_BODY="$(echo "$DOCKER_STATUS" | awk '{print $1": "$2}' | sed 's/^/ /' | sed 's/&/\&/g;s//\>/g' | tr '\n' '
')" +else + DOCKER_BODY="$(echo "$DOCKER_STATUS" | awk '{print $1" "$2}' | sed 's/&/\&/g;s//\>/g' | tr '\n' '
')" +fi -# SSL 证书 +# ---------- SSL 证书 ---------- SSL_EXPIRY=$(echo "" | openssl s_client -connect xmclassmate.top:443 -servername xmclassmate.top 2>&1 | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2 || echo "检测失败") -SSL_DAYS=$(( ($(date -d "$SSL_EXPIRY" +%s 2>/dev/null) - $(date +%s)) / 86400 )) || SSL_DAYS="?" +if [ "$SSL_EXPIRY" != "检测失败" ]; then + SSL_DAYS=$(( ($(date -d "$SSL_EXPIRY" +%s 2>/dev/null) - $(date +%s)) / 86400 )) +else + SSL_DAYS=9999 +fi -# 服务状态 +# ---------- 服务状态 ---------- GREEN_STATUS=$(systemctl is-active rust-backend-green.service 2>/dev/null || echo "unknown") BLUE_STATUS=$(systemctl is-active rust-backend-blue.service 2>/dev/null || echo "unknown") NGINX_STATUS=$(systemctl is-active nginx 2>/dev/null || echo "unknown") -# 最近备份 -LAST_BACKUP=$(ls -1t /root/rust/backups/milkydata/ 2>/dev/null | head -1) -BACKUP_AGE="未知" +# ---------- 备份 ---------- +LAST_BACKUP=$(ls -1t "$BACKUP_DIR/" 2>/dev/null | head -1) +BACKUP_AGE_HOURS="" +BACKUP_AGE_TEXT="未知" if [ -n "$LAST_BACKUP" ]; then - BACKUP_TIME=$(stat -c %Y "/root/rust/backups/milkydata/$LAST_BACKUP" 2>/dev/null) + BACKUP_TIME=$(stat -c %Y "$BACKUP_DIR/$LAST_BACKUP" 2>/dev/null) NOW=$(date +%s) - AGE_HOURS=$(( (NOW - BACKUP_TIME) / 3600 )) - if [ "$AGE_HOURS" -lt 24 ]; then - BACKUP_AGE="✅ ${AGE_HOURS}小时前" - elif [ "$AGE_HOURS" -lt 48 ]; then - BACKUP_AGE="⚠️ ${AGE_HOURS}小时前" - else - BACKUP_AGE="❌ ${AGE_HOURS}小时前" - fi + BACKUP_AGE_HOURS=$(( (NOW - BACKUP_TIME) / 3600 )) + BACKUP_AGE_TEXT="${BACKUP_AGE_HOURS} 小时前" fi -# Fail2Ban 封禁统计 +# ---------- 安全 ---------- SSH_BANS=$(sudo fail2ban-client status sshd 2>/dev/null | grep "Currently banned" | awk '{print $NF}' || echo "0") NGINX_BANS=$(sudo fail2ban-client status nginx-botsearch 2>/dev/null | grep "Currently banned" | awk '{print $NF}' || echo "0") - -# 近期错误数（昨天 journalctl） YESTERDAY_ERRORS=$(journalctl -u rust-backend-green.service --since "1 day ago" --priority err --no-pager 2>/dev/null | wc -l) YESTERDAY_WARNS=$(journalctl -u rust-backend-green.service --since "1 day ago" --priority warning --no-pager 2>/dev/null | wc -l) -# 应用统计（昨天活跃用户） +# ---------- 应用概况（昨日） ---------- ACTIVE_USERS=$(journalctl -u rust-backend-green.service --since "1 day ago" --no-pager 2>/dev/null | grep -c "user_id=\|登录成功\|上传成功" || echo "0") API_COUNT=$(journalctl -u rust-backend-green.service --since "1 day ago" --no-pager 2>/dev/null | grep -c "INFO\|WARN\|ERROR" || echo "0") -# ---------- 构建邮件正文 ---------- +# ---------- 支付与会员（数据库统计，不可用则 N/A） ---------- +# 服务器上通过 docker exec 访问 PostgreSQL（与 check-members.sh 同模式） +DB_QUERY() { + docker exec "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -At -c "$1" 2>/dev/null || echo "" +} +DB_OK=false +if [ -n "$(DB_QUERY 'SELECT 1')" ]; then DB_OK=true; fi -SUBJECT="[ASD 服务器报告] ${DATE} — ${HOST}" +PAY_ORDERS_TODAY="N/A"; PAY_PAID_TODAY="N/A"; PAY_STUCK="N/A" +MEM_ACTIVE="N/A"; MEM_EXPIRE_7D="N/A" +if [ "$DB_OK" = true ]; then + PAY_ORDERS_TODAY=$(DB_QUERY "SELECT count(*) FROM payment_orders WHERE created_at::date = CURRENT_DATE") + PAY_PAID_TODAY=$(DB_QUERY "SELECT count(*) FROM payment_orders WHERE status='paid' AND paid_at::date = CURRENT_DATE") + PAY_STUCK=$(DB_QUERY "SELECT count(*) FROM payment_orders WHERE status='pending' AND created_at < NOW() - INTERVAL '30 minutes'") + MEM_ACTIVE=$(DB_QUERY "SELECT count(*) FROM users WHERE is_member AND (membership_expires_at IS NULL OR membership_expires_at > NOW())") + MEM_EXPIRE_7D=$(DB_QUERY "SELECT count(*) FROM users WHERE is_member AND membership_expires_at IS NOT NULL AND membership_expires_at BETWEEN NOW() AND NOW() + INTERVAL '7 days'") +fi +[ -z "$PAY_ORDERS_TODAY" ] && PAY_ORDERS_TODAY="N/A" +[ -z "$PAY_PAID_TODAY" ] && PAY_PAID_TODAY="N/A" +[ -z "$PAY_STUCK" ] && PAY_STUCK="N/A" +[ -z "$MEM_ACTIVE" ] && MEM_ACTIVE="N/A" +[ -z "$MEM_EXPIRE_7D" ] && MEM_EXPIRE_7D="N/A" -MAIL=$(cat < <当前值> <级别> <附加说明?> +row() { + echo "$1$2$(badge "$3" "$4")" +} +section_title() { + echo "$1" +} -📦 容器 -$(echo "$DOCKER_STATUS" | sed 's/^/ /') +STYLE="font-family:-apple-system,'PingFang SC','Microsoft YaHei',sans-serif;background:#fff;max-width:640px;margin:0 auto;padding:16px" +TH="padding:8px 10px;background:#f1f5f9;color:#64748b;text-align:left;font-size:12px;border-bottom:2px solid #e2e8f0" -🔐 SSL 证书 - 域名: xmclassmate.top - 到期: ${SSL_EXPIRY} - 剩余: ${SSL_DAYS} 天 +BODY=$(cat < +

ASD 服务器健康报告

+
${DATE} · ${HOST} · 发送于 $(date "+%H:%M")
-🔄 服务状态 - - 后端 (green): ${GREEN_STATUS} - - 后端 (blue): ${BLUE_STATUS} - - nginx: ${NGINX_STATUS} + + -🛡 安全 - - SSH 封禁: ${SSH_BANS} 个 IP - - nginx 封禁: ${NGINX_BANS} 个 IP - - 昨日错误: ${YESTERDAY_ERRORS} - - 昨日警告: ${YESTERDAY_WARNS} + $(section_title "🖥 系统状态") + $(row "运行时长" "$UPTIME" ok "OK") + $(row "系统负载" "$LOAD" "$([ "$LOAD1_INT" -ge 4 ] && echo crit || { [ "$LOAD1_INT" -ge 2 ] && echo warn || echo ok; })" "$([ "$LOAD1_INT" -ge 4 ] && echo "超载" || { [ "$LOAD1_INT" -ge 2 ] && echo "偏高" || echo "正常"; })") + $(row "磁盘使用" "${DISK_USED} / ${DISK_TOTAL} (${DISK_PCT_STR})" "$(pct_level "$DISK_PCT")" "$(pct_level "$DISK_PCT" | tr 'a-z' 'A-Z')") + $(row "内存使用" "${MEM_USED} / ${MEM_TOTAL} (${MEM_PCT}%)" "$(pct_level "$MEM_PCT" 80 90)" "$(pct_level "$MEM_PCT" 80 90 | tr 'a-z' 'A-Z')") + $(row "CPU 温度" "${CPU_TEMP}" ok "OK") -📊 应用概况（昨日） - - 日志事件数: ${API_COUNT} - - 活跃用户操作: ${ACTIVE_USERS} + $(section_title "📦 容器与服务") + $(row "Docker 容器" "运行中 $(echo "$DOCKER_STATUS" | grep -c "Up " || echo 0) 个 / 异常 $(echo "$DOCKER_STATUS" | grep -vc "Up " || echo 0) 个" "$([ "$DOCKER_DOWN" -gt 0 ] && echo crit || echo ok)" "$([ "$DOCKER_DOWN" -gt 0 ] && echo "有异常" || echo "正常")") + $(row "后端 green" "$GREEN_STATUS" "$([ "$GREEN_STATUS" = "active" ] && echo ok || echo crit)" "$([ "$GREEN_STATUS" = "active" ] && echo "运行中" || echo "异常")") + $(row "后端 blue" "$BLUE_STATUS" "$([ "$BLUE_STATUS" = "active" ] && echo ok || echo crit)" "$([ "$BLUE_STATUS" = "active" ] && echo "运行中" || echo "异常")") + $(row "nginx" "$NGINX_STATUS" "$([ "$NGINX_STATUS" = "active" ] && echo ok || echo crit)" "$([ "$NGINX_STATUS" = "active" ] && echo "运行中" || echo "异常")") + -💾 备份 - - 最新备份: ${LAST_BACKUP:-无} - - 备份时效: ${BACKUP_AGE} + $(section_title "🔐 SSL 与安全") + $(row "SSL 证书" "${SSL_EXPIRY}（剩余 ${SSL_DAYS} 天）" "$([ "$SSL_DAYS" -lt 7 ] && echo crit || { [ "$SSL_DAYS" -lt 30 ] && echo warn || echo ok; })" "$([ "$SSL_DAYS" -lt 7 ] && echo "即将过期" || { [ "$SSL_DAYS" -lt 30 ] && echo "注意续期" || echo "正常"; })") + $(row "Fail2ban SSH" "${SSH_BANS} 个 IP 被封禁" ok "OK") + $(row "Fail2ban nginx" "${NGINX_BANS} 个 IP 被封禁" ok "OK") + $(row "昨日日志错误" "$YESTERDAY_ERRORS 条" "$([ "$YESTERDAY_ERRORS" -gt 50 ] && echo warn || echo ok)" "$([ "$YESTERDAY_ERRORS" -gt 50 ] && echo "偏多" || echo "正常")") + $(row "昨日日志警告" "$YESTERDAY_WARNS 条" "$([ "$YESTERDAY_WARNS" -gt 200 ] && echo warn || echo ok)" "$([ "$YESTERDAY_WARNS" -gt 200 ] && echo "偏多" || echo "正常")") ---- -ASD 自动监控 | $(date "+%Y-%m-%d %H:%M") + $(section_title "💾 备份") + $(row "最新备份" "${LAST_BACKUP:-无}" "$([ -n "$BACKUP_AGE_HOURS" ] && { [ "$BACKUP_AGE_HOURS" -ge 48 ] && echo crit || { [ "$BACKUP_AGE_HOURS" -ge 24 ] && echo warn || echo ok; }; } || echo crit)" "$([ -n "$BACKUP_AGE_HOURS" ] && { [ "$BACKUP_AGE_HOURS" -ge 48 ] && echo "严重过期" || { [ "$BACKUP_AGE_HOURS" -ge 24 ] && echo "⚠ 超24小时" || echo "${BACKUP_AGE_TEXT}"; }; } || echo "未知")") + $(row "备份时效" "${BACKUP_AGE_TEXT}" ok "OK") + + $(section_title "📊 应用概况（昨日）") + $(row "日志事件数" "$API_COUNT 条" ok "OK") + $(row "活跃用户操作" "$ACTIVE_USERS 次" ok "OK") + + $(section_title "💰 支付与会员") + $(row "今日新订单" "$PAY_ORDERS_TODAY 笔" ok "OK") + $(row "今日支付成功" "$PAY_PAID_TODAY 笔" ok "OK") + $(row "卡单（pending>30分钟）" "$PAY_STUCK 笔" "$([ "$PAY_STUCK" -gt 0 ] 2>/dev/null && echo crit || echo ok)" "$([ "$PAY_STUCK" -gt 0 ] 2>/dev/null && echo "需人工处理" || echo "无")") + $(row "活跃会员" "$MEM_ACTIVE 人" ok "OK") + $(row "7天内到期会员" "$MEM_EXPIRE_7D 人" "$([ "$MEM_EXPIRE_7D" -gt 0 ] 2>/dev/null && echo warn || echo ok)" "$([ "$MEM_EXPIRE_7D" -gt 0 ] 2>/dev/null && echo "可发续费提醒" || echo "无")") + $( [ "$DB_OK" = false ] && echo "" ) +
指标当前值状态
${DOCKER_BODY}
⚠ 数据库不可用，支付/会员统计缺失
+ +
ASD 自动监控 | $(date "+%Y-%m-%d %H:%M") | 异常项请关注 alert.sh 实时告警
+ EOF ) -# ---------- 发送邮件 ---------- - -echo "$MAIL" | msmtp -a default "$TO" -echo "✅ 健康报告已发送到 ${TO}" +# ---------- 发送 ---------- +if mail_send "$SUBJECT" "$BODY"; then + echo "✅ 健康报告已发送到 ${MAIL_TO}（$(date '+%H:%M:%S')）" +else + echo "❌ 健康报告发送失败，详见 ${MAIL_LOG_DIR}/mail.log" + exit 1 +fi diff --git a/scripts/mail-lib.sh b/scripts/mail-lib.sh new file mode 100644 index 0000000..c941a3c --- /dev/null +++ b/scripts/mail-lib.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# =========================================== +# mail-lib.sh — 邮件发送共享库 +# 供 daily-report.sh / alert.sh 复用： +# mail_send <主题> （带重试 + 日志 + 轮转） +# =========================================== + +# ---------- 可配置项（环境变量可覆盖） ---------- +: "${MAIL_TO:=m943790568@163.com}" # 收件人，多个用逗号分隔 +: "${MAIL_FROM:=ASD 监控 }" # 发件人显示名 +: "${MAIL_ACCOUNT:=default}" # msmtp 账户（~/.msmtprc 中的 account） +: "${MAIL_LOG_DIR:=/root/rust/scripts/logs}" # 日志目录 +: "${MAIL_RETRY:=3}" # 发送重试次数 +: "${MAIL_LOG_MAX_KB:=1024}" # mail.log 轮转阈值（KB） + +mkdir -p "$MAIL_LOG_DIR" 2>/dev/null || true + +# 写日志 +mail_log() { + local level="$1"; shift + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [${level}] $*" >> "${MAIL_LOG_DIR}/mail.log" 2>/dev/null || true +} + +# 日志轮转：超过阈值保留末尾 200 行 +mail_rotate_log() { + local f="${MAIL_LOG_DIR}/mail.log" + [ -f "$f" ] || return 0 + local size_kb=$(( $(stat -c %s "$f" 2>/dev/null || echo 0) / 1024 )) + if [ "$size_kb" -gt "$MAIL_LOG_MAX_KB" ]; then + tail -n 200 "$f" > "$f.tmp" 2>/dev/null && mv "$f.tmp" "$f" 2>/dev/null || true + mail_log INFO "mail.log 已轮转" + fi +} + +# 发送 HTML 邮件（UTF-8），失败自动重试 +# 用法: mail_send "主题" "HTML 正文" +mail_send() { + local subject="$1" body="$2" + mail_rotate_log + + local recipients headers + recipients=$(echo "$MAIL_TO" | tr ',' ' ') + headers=$(cat <>"${MAIL_LOG_DIR}/mail.log"; then + ok=1 + mail_log INFO "邮件发送成功: ${subject} → ${MAIL_TO}（第 ${attempt} 次尝试）" + break + fi + mail_log WARN "邮件发送失败（第 ${attempt}/${MAIL_RETRY} 次）: ${subject}" + attempt=$((attempt + 1)) + [ "$attempt" -le "$MAIL_RETRY" ] && sleep $((attempt * 5)) + done + + if [ "$ok" -ne 1 ]; then + mail_log ERROR "邮件最终发送失败: ${subject}" + return 1 + fi + return 0 +}