asd-backend/deploy.sh

#!/bin/bash

# ===========================================
# Rust Backend Deployment Script
# ===========================================
# Usage: ./deploy.sh [development|production] [options]
#
# Options:
#   --dry-run      预览模式，不执行实际操作
#   --yes, -y     跳过确认提示
#   --skip-tests   跳过部署后测试
#   --rollback     回滚到上一个备份版本
#   --backup-list  列出可用备份
# ===========================================

set -euo pipefail

PROJECT_NAME="rust-backend"
REMOTE_USER="root"
REMOTE_HOST="1panel-server"
APP_ENV="production"

# 默认选项
DRY_RUN=false
SKIP_CONFIRM=false
SKIP_TESTS=false
ROLLBACK=false
LIST_BACKUPS=false
VIEW_LOGS=false
TAIL_LOGS=false
LOG_LINES=50
VIEW_STATUS=false
POST_DEPLOY_LOGS=false

# 全局变量（部署流程中使用）
BACKUP_PATH=""

# 解析选项（先处理选项，再处理位置参数）
while [[ $# -gt 0 ]]; do
    case "$1" in
        --dry-run)
            DRY_RUN=true
            ;;
        --yes|-y)
            SKIP_CONFIRM=true
            ;;
        --skip-tests)
            SKIP_TESTS=true
            ;;
        --rollback)
            ROLLBACK=true
            ;;
        --backup-list)
            LIST_BACKUPS=true
            ;;
        --logs)
            VIEW_LOGS=true
            ;;
        --tail)
            TAIL_LOGS=true
            ;;
        --logs=*)
            VIEW_LOGS=true
            LOG_LINES="${1#*=}"
            ;;
        --tail=*)
            TAIL_LOGS=true
            LOG_LINES="${1#*=}"
            ;;
        --status)
            VIEW_STATUS=true
            ;;
        --post-logs)
            POST_DEPLOY_LOGS=true
            ;;
        --help|-h)
            echo "用法: $0 [development|production] [options]"
            echo ""
            echo "选项:"
            echo "  --dry-run      预览模式，不执行实际操作"
            echo "  --yes, -y      跳过确认提示"
            echo "  --skip-tests   跳过部署后测试"
            echo "  --rollback     回滚到上一个备份版本"
            echo "  --backup-list  列出可用备份"
            echo "  --logs [N]     查看后端日志（默认50行）"
            echo "  --tail [N]    实时跟踪后端日志（Ctrl+C 退出）"
            echo "  --status      查看服务状态"
            echo "  --post-logs    部署后自动显示日志"
            exit 0
            ;;
        *)
            # 位置参数：development 或 production
            if [[ "$1" == "development" || "$1" == "production" ]]; then
                APP_ENV="$1"
            else
                echo "未知选项: $1"
                echo "用法: $0 [development|production] [options]"
                exit 1
            fi
            ;;
    esac
    shift
done

# 根据环境选择远程目录和服务
case "${APP_ENV}" in
    development)
        REMOTE_DIR="/root/rust/rust_backend_dev"
        SERVICE_NAME="rust-backend-dev.service"
        TEST_DOMAIN="https://dev.xmclassmate.top"
        BACKEND_PORT="8080"
        DB_CONTAINER="1Panel-postgresql-FtMo"
        DB_NAME="milkydata_dev"
        DB_USER="milkydata"
        ;;
    production)
        REMOTE_DIR="/root/rust/rust_backend"
        SERVICE_NAME="rust-backend.service"
        TEST_DOMAIN="https://xmclassmate.top"
        BACKEND_PORT="4433"
        DB_CONTAINER="1Panel-postgresql-FtMo"
        DB_NAME="milkydata"
        DB_USER="milkydata"
        ;;
    *)
        echo "无效的环境: ${APP_ENV}"
        echo "支持的環境: development, production"
        exit 1
        ;;
esac

# 彩色输出
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1" >&2; }
log_step() { echo -e "${BLUE}[STEP]${NC} $1"; }
log_dry() { echo -e "${YELLOW}[DRY-RUN]${NC} $1"; }

# 获取 Git 版本信息
get_git_info() {
    local git_branch
    local git_commit
    local git_version

    git_branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
    git_commit=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
    git_version="${git_branch}@${git_commit}"

    echo "$git_version"
}

# 部署日志路径
get_deploy_log() {
    echo "${REMOTE_DIR}/deploy.log"
}

# 记录部署日志
log_to_file() {
    if [ "$DRY_RUN" = true ]; then
        return 0
    fi

    local timestamp
    timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    local log_entry="[${timestamp}] ${APP_ENV} ${GIT_VERSION} $1"

    ssh "${REMOTE_USER}@${REMOTE_HOST}" "echo '${log_entry}' >> $(get_deploy_log)" 2>/dev/null || true
}

# Webhook 通知
send_webhook() {
    local status="$1"
    local message="$2"

    if [ -z "${WEBHOOK_URL:-}" ]; then
        return 0
    fi

    local payload
    payload=$(cat <<EOF
{
    "app": "rust-backend",
    "env": "${APP_ENV}",
    "status": "${status}",
    "version": "${GIT_VERSION}",
    "message": "${message}",
    "timestamp": "$(date -Iseconds)"
}
EOF
)

    if [ "$DRY_RUN" = true ]; then
        log_dry "curl -X POST '${WEBHOOK_URL}' -d '${payload}'"
        return 0
    fi

    curl -s -X POST "${WEBHOOK_URL}" \
        -H "Content-Type: application/json" \
        -d "${payload}" >/dev/null 2>&1 || true
}

# 列出备份
list_backups() {
    log_step "可用备份列表..."
    echo ""

    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'ls -la ${REMOTE_DIR}/${PROJECT_NAME}.backup.*'"
        return 0
    fi

    ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls -la ${REMOTE_DIR}/${PROJECT_NAME}.backup.*" 2>/dev/null || echo "无备份文件"
    echo ""
}

# 查看后端日志
view_logs() {
    local lines="${LOG_LINES:-50}"
    log_step "查看后端日志（最后 ${lines} 行）..."
    echo ""

    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'tail -${lines} ${REMOTE_DIR}/logs/rust-backend.*.log'"
        return 0
    fi

    # 获取最新的日志文件
    local log_file
    log_file=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls -t ${REMOTE_DIR}/logs/rust-backend.*.log 2>/dev/null | head -1" || echo "")

    if [ -z "$log_file" ]; then
        log_warn "未找到日志文件: ${REMOTE_DIR}/logs/"
        return 1
    fi

    echo -e "${BLUE}=== ${log_file} ===${NC}"
    echo ""
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "tail -${lines} \"${log_file}\""
    echo ""
}

# 实时跟踪后端日志
tail_logs() {
    local lines="${LOG_LINES:-100}"
    log_step "实时跟踪后端日志（最后 ${lines} 行，按 Ctrl+C 退出）..."
    echo ""

    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'tail -f -n ${lines} ${REMOTE_DIR}/logs/rust-backend.*.log'"
        return 0
    fi

    # 获取最新的日志文件
    local log_file
    log_file=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls -t ${REMOTE_DIR}/logs/rust-backend.*.log 2>/dev/null | head -1" || echo "")

    if [ -z "$log_file" ]; then
        log_warn "未找到日志文件: ${REMOTE_DIR}/logs/"
        return 1
    fi

    echo -e "${BLUE}=== 实时日志: ${log_file} ===${NC}"
    echo -e "${YELLOW}按 Ctrl+C 退出${NC}"
    echo ""
    ssh -t "${REMOTE_USER}@${REMOTE_HOST}" "tail -f -n ${lines} \"${log_file}\""
}

# 查看服务状态
status() {
    log_step "服务状态..."
    echo ""

    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'systemctl status ${SERVICE_NAME}'"
        return 0
    fi

    echo -e "${BLUE}=== ${SERVICE_NAME} ===${NC}"
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl status ${SERVICE_NAME} --no-pager"
    echo ""

    echo -e "${BLUE}=== 最近日志 ===${NC}"
    local log_file
    log_file=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls -t ${REMOTE_DIR}/logs/rust-backend.*.log 2>/dev/null | head -1" || echo "")
    if [ -n "$log_file" ]; then
        echo -e "${BLUE}=== ${log_file} ===${NC}"
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "tail -20 \"${log_file}\""
    fi
    echo ""
}

# 回滚（手动触发）
rollback() {
    log_step "查找最新备份..."
    local latest_backup

    if [ "$DRY_RUN" = true ]; then
        latest_backup="${REMOTE_DIR}/${PROJECT_NAME}.backup.20240101_120000"
        log_dry "latest_backup=${latest_backup}"
    else
        latest_backup=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "ls -t ${REMOTE_DIR}/${PROJECT_NAME}.backup.* 2>/dev/null | head -1" || echo "")

        if [ -z "$latest_backup" ]; then
            log_error "没有找到可用的备份"
            return 1
        fi
    fi

    echo ""
    echo -e "${YELLOW}========== 回滚确认 ==========${NC}"
    echo -e "  将回滚到: ${latest_backup}"
    echo -e "  服务: ${SERVICE_NAME}"
    echo -e "================================${NC}"
    echo ""

    if [ "$SKIP_CONFIRM" = false ]; then
        read -p "确认回滚? [y/N] " -n 1 -r
        echo
        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
            log_warn "回滚已取消"
            return 0
        fi
    fi

    log_step "停止服务..."
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl stop ${SERVICE_NAME}" 2>/dev/null || log_warn "停止服务失败，继续尝试..."

    log_step "恢复备份..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'cp ${latest_backup} ${REMOTE_DIR}/${PROJECT_NAME}'"
    else
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "cp ${latest_backup} ${REMOTE_DIR}/${PROJECT_NAME}"
    fi

    log_step "启动服务..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'systemctl start ${SERVICE_NAME}'"
    else
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl start ${SERVICE_NAME}"
    fi

    log_info "回滚完成"
    log_to_file "MANUAL_ROLLBACK to $(basename $latest_backup)"
    send_webhook "rollback" "手动回滚完成: $(basename $latest_backup)"
}

# 回滚到上一个正常版本（测试失败时自动触发）
rollback_on_failure() {
    local backup_path="$1"

    if [ -z "$backup_path" ]; then
        log_error "回滚失败: 未指定备份路径"
        return 1
    fi

    echo ""
    log_error "========== 测试失败，自动回滚 =========="
    echo -e "  原因: 部署后测试未通过"
    echo -e "  回滚到: ${backup_path}"
    echo -e "  服务: ${SERVICE_NAME}"
    echo -e "=========================================="
    echo ""

    log_step "停止服务..."
    if [ "$DRY_RUN" = false ]; then
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl stop ${SERVICE_NAME}" 2>/dev/null || log_warn "停止服务失败，继续尝试..."
    else
        log_dry "systemctl stop ${SERVICE_NAME}"
    fi

    log_step "恢复上一个正常版本..."
    if [ "$DRY_RUN" = false ]; then
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "cp '${backup_path}' '${REMOTE_DIR}/${PROJECT_NAME}'"
    else
        log_dry "cp '${backup_path}' '${REMOTE_DIR}/${PROJECT_NAME}'"
    fi

    log_step "启动服务..."
    if [ "$DRY_RUN" = false ]; then
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl start ${SERVICE_NAME}"
        sleep 3
    else
        log_dry "systemctl start ${SERVICE_NAME}"
    fi

    log_info "自动回滚完成"
    log_to_file "AUTO_ROLLBACK to $(basename $backup_path) - tests failed"
    send_webhook "failure" "部署测试失败，已自动回滚到: $(basename $backup_path)"
}

# 清理旧备份，只保留最近 N 个
cleanup_old_backups() {
    local keep_count="${1:-5}"

    if [ "$DRY_RUN" = true ]; then
        log_dry "保留最近 ${keep_count} 个备份，删除其余"
        return 0
    fi

    local backup_count
    backup_count=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "ls -t ${REMOTE_DIR}/${PROJECT_NAME}.backup.* 2>/dev/null | wc -l" || echo "0")

    if [ "$backup_count" -le "$keep_count" ]; then
        return 0
    fi

    local to_delete
    to_delete=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "ls -t ${REMOTE_DIR}/${PROJECT_NAME}.backup.* 2>/dev/null | tail -n +$((keep_count + 1))" || echo "")

    if [ -z "$to_delete" ]; then
        return 0
    fi

    log_step "清理旧备份（保留 ${keep_count} 个）..."
    echo "$to_delete" | while read -r file; do
        if [ -n "$file" ]; then
            ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f '${file}'" 2>/dev/null && echo "  已删除: ${file}" || true
        fi
    done
}

# 检查依赖
check_dependencies() {
    log_step "检查依赖..."
    local deps=("cargo" "rsync" "ssh")
    for cmd in "${deps[@]}"; do
        if ! command -v "$cmd" &> /dev/null; then
            log_error "依赖命令 '$cmd' 未找到，请先安装。"
        fi
    done
    log_info "依赖检查通过"
}

# 获取远程服务器 SSH ControlMaster socket 路径
get_ssh_socket() {
    echo "/tmp/ssh-master-${REMOTE_HOST}-$$"
}

# 建立 SSH 连接（复用）
ssh_connect() {
    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh -o ControlMaster=auto -o ControlPath=$(get_ssh_socket) -o ControlPersist=5 ${REMOTE_USER}@${REMOTE_HOST}"
        return 0
    fi

    local socket
    socket=$(get_ssh_socket)

    # 检查是否已有连接
    if ! ssh -o "ControlPath=${socket}" -o "ConnectionAttempts=3" -o "ConnectTimeout=10" \
        -o "ControlMaster=no" -o "ControlPersist=no" \
        "${REMOTE_USER}@${REMOTE_HOST}" "echo 'connection test'" &>/dev/null; then

        log_info "建立 SSH 连接..."
        ssh -o ControlMaster=auto -o ControlPath="${socket}" -o ControlPersist=600 \
            -o ConnectionAttempts=3 -o ConnectTimeout=10 \
            -N "${REMOTE_USER}@${REMOTE_HOST}" &
        sleep 2
    fi
}

# 关闭 SSH 连接
ssh_disconnect() {
    if [ "$DRY_RUN" = true ]; then
        return 0
    fi

    local socket
    socket=$(get_ssh_socket)
    ssh -o "ControlPath=${socket}" -O exit "${REMOTE_USER}@${REMOTE_HOST}" 2>/dev/null || true
}

# 编译项目
build_project() {
    log_step "编译项目 (release模式)..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "APP_ENV=\"${APP_ENV}\" cargo build --release"
        return 0
    fi

    if APP_ENV="${APP_ENV}" cargo build --release; then
        log_info "编译成功"
    else
        log_error "编译失败！"
    fi
}

# 备份旧版本并保存回滚标记
backup_old_version() {
    log_step "备份旧版本..."
    local binary_path="./target/release/${PROJECT_NAME}"
    local timestamp
    timestamp=$(date +%Y%m%d_%H%M%S)
    BACKUP_PATH="${REMOTE_DIR}/${PROJECT_NAME}.backup.${timestamp}"

    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'if [ -f ${REMOTE_DIR}/${PROJECT_NAME} ]; then cp ${REMOTE_DIR}/${PROJECT_NAME} ${BACKUP_PATH}; echo \"\${BACKUP_PATH}\" > ${REMOTE_DIR}/.last_deployed; fi'"
        return 0
    fi

    ssh "${REMOTE_USER}@${REMOTE_HOST}" "
        if [ -f '${REMOTE_DIR}/${PROJECT_NAME}' ]; then
            cp '${REMOTE_DIR}/${PROJECT_NAME}' '${BACKUP_PATH}'
            echo '${BACKUP_PATH}' > '${REMOTE_DIR}/.last_deployed'
            echo '已备份到: ${BACKUP_PATH}'
        else
            echo '无旧版本需要备份'
        fi
    "
    log_info "备份完成"
}

# 上传二进制文件
upload_binary() {
    log_step "上传二进制文件..."
    local binary_path="./target/release/${PROJECT_NAME}"

    if [ ! -f "$binary_path" ]; then
        log_error "编译产物未找到: $binary_path"
    fi

    if [ "$DRY_RUN" = true ]; then
        log_dry "rsync -avzP '$binary_path' '${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/'"
        return 0
    fi

    rsync -avzP --progress "$binary_path" "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/"
    log_info "上传完成"
}

# 上传配置文件（只上传 .toml 文件）
upload_config() {
    log_step "上传配置文件..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "rsync -avzP --include='*.toml' --exclude='*' ./config/ '${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/config/'"
        return 0
    fi

    rsync -avzP --include='*.toml' --exclude='*' ./config/ "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/config/"
    log_info "配置文件上传完成"
}

# 上传迁移文件
upload_migrations() {
    log_step "上传迁移文件..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "rsync -avzP ./migrations/ '${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/migrations/'"
        return 0
    fi

    if [ ! -d "./migrations" ]; then
        log_warn "迁移目录不存在，跳过"
        return 0
    fi

    rsync -avzP --progress ./migrations/ "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/migrations/"
    log_info "迁移文件上传完成"
}

# 上传测试脚本
upload_test_script() {
    log_step "上传测试脚本..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "rsync -avzP ./test_deployment.sh '${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/'"
        return 0
    fi

    rsync -avzP --progress ./test_deployment.sh "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/"
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "chmod +x ${REMOTE_DIR}/test_deployment.sh"
    log_info "测试脚本上传完成"
}

# 上传清理脚本（定时任务用）
upload_cleanup_script() {
    log_step "上传清理脚本..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "rsync -avzP ./scripts/cleanup_refresh_tokens.sh '${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/scripts/'"
        return 0
    fi

    mkdir -p ./scripts 2>/dev/null || true
    if [ -f "./scripts/cleanup_refresh_tokens.sh" ]; then
        rsync -avzP --progress ./scripts/cleanup_refresh_tokens.sh "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/scripts/"
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "chmod +x ${REMOTE_DIR}/scripts/cleanup_refresh_tokens.sh"
        log_info "清理脚本上传完成"
    else
        log_warn "清理脚本不存在，跳过"
    fi
}

# 重启远程服务
restart_service() {
    log_step "重启服务 '${SERVICE_NAME}'..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'systemctl restart ${SERVICE_NAME}'"
        return 0
    fi

    if ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl restart ${SERVICE_NAME}"; then
        log_info "服务重启成功"
    else
        log_error "服务重启失败！请检查服务状态和日志。"
    fi

    # 等待服务启动
    sleep 3
}

run_tests() {
    log_step "执行部署后测试..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "完整测试套件:"
        log_dry "  1. test_basic_connectivity - 基础连通性"
        log_dry "  2. test_database_connection - 数据库连接"
        log_dry "  3. test_token_flow - Token 完整流程"
        log_dry "  4. test_weather_details_jwt - weather/details JWT 测试"
        log_dry "  5. test_invalid_token_401 - 无效 Token 401 测试"
        log_dry "  6. test_response_content - 响应内容验证"
        return 0
    fi

    if [ "$SKIP_TESTS" = true ]; then
        log_warn "跳过测试（--skip-tests）"
        return 0
    fi

    local failed=0

    log_step "测试 1/6: 基础连通性..."
    if ! test_basic_connectivity; then
        rollback_and_exit "基础连通性测试失败"
    fi

    log_step "测试 2/6: 数据库连接..."
    if ! test_database_connection; then
        rollback_and_exit "数据库连接测试失败"
    fi

    log_step "测试 3/6: Token 完整流程..."
    if ! test_token_flow; then
        rollback_and_exit "Token 流程测试失败"
    fi

    log_step "测试 4/6: weather/details JWT 测试..."
    if ! test_weather_details_jwt; then
        rollback_and_exit "weather/details JWT 测试失败"
    fi

    log_step "测试 5/6: 无效 Token 401 测试..."
    if ! test_invalid_token_401; then
        rollback_and_exit "无效 Token 401 测试失败"
    fi

    log_step "测试 6/6: 响应内容验证..."
    if ! test_response_content; then
        rollback_and_exit "响应内容验证失败"
    fi

    log_info "========== 所有测试通过 =========="
    return 0
}

test_basic_connectivity() {
    local max_retries=3
    local retry=0

    while [ $retry -lt $max_retries ]; do
        retry=$((retry + 1))
        log_info "尝试 ${retry}/${max_retries}: 测试本地后端 (127.0.0.1:${BACKEND_PORT})"

        local all_passed=true

        log_info "  [1/4] 测试 /health..."
        local health_code
        health_code=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 http://127.0.0.1:${BACKEND_PORT}/health" 2>/dev/null || echo "000")
        if [ "$health_code" != "200" ]; then
            log_warn "  /health 异常 (HTTP ${health_code})"
            all_passed=false
        else
            log_info "  /health 正常"
        fi

        log_info "  [2/4] 测试 /api/login..."
        local login_code
        login_code=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 -X POST http://127.0.0.1:${BACKEND_PORT}/api/login -H 'Content-Type: application/json' -d '{\"code\":\"test_code\"'" 2>/dev/null || echo "000")
        if [ "$login_code" != "400" ] && [ "$login_code" != "200" ]; then
            log_warn "  /api/login 异常 (HTTP ${login_code})"
            all_passed=false
        else
            log_info "  /api/login 正常 (HTTP ${login_code})"
        fi

        log_info "  [3/4] 测试 /api/refresh-token..."
        local refresh_code
        refresh_code=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 -X POST http://127.0.0.1:${BACKEND_PORT}/api/refresh-token -H 'Content-Type: application/json' -d '{\"refresh_token\":\"invalid\"}'" 2>/dev/null || echo "000")
        if [ "$refresh_code" != "401" ] && [ "$refresh_code" != "200" ]; then
            log_warn "  /api/refresh-token 异常 (HTTP ${refresh_code})"
            all_passed=false
        else
            log_info "  /api/refresh-token 正常 (HTTP ${refresh_code})"
        fi

        log_info "  [4/4] 测试 / (根路径)..."
        local root_code
        root_code=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 http://127.0.0.1:${BACKEND_PORT}/" 2>/dev/null || echo "000")
        if [ "$root_code" != "200" ] && [ "$root_code" != "404" ]; then
            log_warn "  / 异常 (HTTP ${root_code})"
            all_passed=false
        else
            log_info "  / 正常 (HTTP ${root_code})"
        fi

        if [ "$all_passed" = true ]; then
            log_info "基础连通性测试通过"
            return 0
        fi

        if [ $retry -lt $max_retries ]; then
            sleep 2
        fi
    done

    log_error "基础连通性测试失败"
    return 1
}

test_database_connection() {
    log_info "  测试 Docker 容器连接..."
    local ping_result
    ping_result=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "docker exec ${DB_CONTAINER} pg_isready -U ${DB_USER} 2>/dev/null" || echo "failed")
    if [[ ! "$ping_result" =~ "accepting connections" ]]; then
        log_warn "  PostgreSQL 无法连接: ${ping_result}"
        return 1
    fi
    log_info "  PostgreSQL 连接正常"

    log_info "  测试数据库查询..."
    local query_result
    query_result=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "docker exec ${DB_CONTAINER} psql -U ${DB_USER} -d ${DB_NAME} -c 'SELECT 1;' 2>/dev/null" || echo "failed")
    if [[ ! "$query_result" =~ "1 row" ]]; then
        log_warn "  数据库查询失败: ${query_result}"
        return 1
    fi
    log_info "  数据库查询正常"

    log_info "  测试表存在..."
    local tables_result
    tables_result=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "docker exec ${DB_CONTAINER} psql -U ${DB_USER} -d ${DB_NAME} -c 'SELECT tablename FROM pg_tables WHERE schemaname = 'public' LIMIT 5;' 2>/dev/null" || echo "failed")
    if [[ ! "$tables_result" =~ "users" ]]; then
        log_warn "  users 表不存在"
        return 1
    fi
    log_info "  users 表存在"

    log_info "数据库连接测试通过"
    return 0
}

test_token_flow() {
    local max_retries=3
    local retry=0

    while [ $retry -lt $max_retries ]; do
        retry=$((retry + 1))
        log_info "尝试 ${retry}/${max_retries}: Token 完整流程测试"

        log_info "  [1/3] 登录获取 Token..."
        local login_response
        login_response=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s --max-time 10 -X POST http://127.0.0.1:${BACKEND_PORT}/api/login -H 'Content-Type: application/json' -d '{\"code\":\"test_code_12345\"'" 2>/dev/null || echo "")

        if [ -z "$login_response" ]; then
            log_warn "  登录响应为空"
            sleep 2
            continue
        fi

        local token
        token=$(echo "$login_response" | grep -o '"token":"[^"]*"' | head -1 | cut -d'"' -f4 || echo "")
        local refresh_token
        refresh_token=$(echo "$login_response" | grep -o '"refresh_token":"[^"]*"' | head -1 | cut -d'"' -f4 || echo "")

        if [ -z "$token" ]; then
            log_warn "  未获取到 token（微信 code 无效是正常的）"
            log_info "  但 refresh_token 端点仍然可测试"
        else
            log_info "  获取到 token: ${token:0:50}..."
        fi

        log_info "  [2/3] 测试 /api/user/profile (受保护接口)..."
        if [ -z "$token" ]; then
            log_warn "  跳过：无可用 token"
        else
            local profile_response
            profile_response=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
                "curl -s --max-time 10 http://127.0.0.1:${BACKEND_PORT}/api/user/profile -H 'Authorization: Bearer ${token}'" 2>/dev/null || echo "")

            local profile_success
            profile_success=$(echo "$profile_response" | grep -o '"success":true' || echo "")
            if [ -n "$profile_success" ]; then
                log_info "  /api/user/profile 正常 (JWT 验证成功)"
            else
                log_warn "  /api/user/profile 异常: ${profile_response:0:100}"
            fi
        fi

        log_info "  [3/3] 测试 refresh_token 端点..."
        local test_refresh_token="MToxMDAwOmRldi1vbmx5LXNlY3JldC1jaGFuZ2UtaW4tcHJvZHVjdGlvbg=="
        local refresh_response
        refresh_response=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s --max-time 10 -X POST http://127.0.0.1:${BACKEND_PORT}/api/refresh-token -H 'Content-Type: application/json' -d '{\"refresh_token\":\"${test_refresh_token}\"}'" 2>/dev/null || echo "")

        local refresh_code
        refresh_code=$(echo "$refresh_response" | grep -o '"success":true' || echo "")
        if [ -n "$refresh_code" ]; then
            log_info "  refresh_token 端点正常"
        else
            log_warn "  refresh_token 端点响应: ${refresh_response:0:100}"
        fi

        log_info "Token 流程测试通过"
        return 0
    done

    log_error "Token 流程测试失败"
    return 1
}

test_weather_details_jwt() {
    local max_retries=3
    local retry=0

    while [ $retry -lt $max_retries ]; do
        retry=$((retry + 1))
        log_info "尝试 ${retry}/${max_retries}: weather/details JWT 测试"

        log_info "  [1/2] 获取有效 Token..."
        local login_response
        login_response=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s --max-time 10 -X POST http://127.0.0.1:${BACKEND_PORT}/api/login -H 'Content-Type: application/json' -d '{\"code\":\"test_code_weather\"'" 2>/dev/null || echo "")

        local token
        token=$(echo "$login_response" | grep -o '"token":"[^"]*"' | head -1 | cut -d'"' -f4 || echo "")

        if [ -z "$token" ]; then
            log_warn "  未获取到 token，跳过 weather/details 测试"
            log_info "  （这是正常的，因为微信 code 无效）"
            return 0
        fi

        log_info "  [2/2] 测试 /weather/details?id=1 带 JWT..."
        local weather_response
        weather_response=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s --max-time 10 'http://127.0.0.1:${BACKEND_PORT}/weather/details?id=1' -H 'Authorization: Bearer ${token}'" 2>/dev/null || echo "")

        local weather_success
        weather_success=$(echo "$weather_response" | grep -o '"success":true' || echo "")
        local weather_data
        weather_data=$(echo "$weather_response" | grep -o '"data":{' || echo "")

        if [ -n "$weather_success" ] || [ -n "$weather_data" ]; then
            log_info "  /weather/details?id=1 JWT 验证成功"
            return 0
        fi

        local weather_error
        weather_error=$(echo "$weather_response" | grep -o '"error":"[^"]*"' | head -1 || echo "")
        local weather_403
        weather_403=$(echo "$weather_response" | grep -o "Forbidden" || echo "")

        if [ -n "$weather_error" ] || [ -n "$weather_403" ]; then
            log_info "  /weather/details?id=1 JWT 验证通过，但资源无权限（正常）"
            return 0
        fi

        log_warn "  /weather/details 响应异常: ${weather_response:0:150}"
        sleep 2
    done

    log_error "weather/details JWT 测试失败"
    return 1
}

test_invalid_token_401() {
    local max_retries=3
    local retry=0

    while [ $retry -lt $max_retries ]; do
        retry=$((retry + 1))
        log_info "尝试 ${retry}/${max_retries}: 无效 Token 401 测试"

        log_info "  [1/3] 测试 /api/user/profile (无效 Token)..."
        local invalid_token="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MDAwMDAwMDB9.invalid_signature"
        local response_401
        response_401=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 http://127.0.0.1:${BACKEND_PORT}/api/user/profile -H 'Authorization: Bearer ${invalid_token}'" 2>/dev/null || echo "000")

        if [ "$response_401" = "401" ]; then
            log_info "  /api/user/profile 无效 Token 返回 401 ✓"
        else
            log_warn "  /api/user/profile 异常 (HTTP ${response_401})"
            sleep 2
            continue
        fi

        log_info "  [2/3] 测试 /weather/details (无效 Token)..."
        local weather_401
        weather_401=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 'http://127.0.0.1:${BACKEND_PORT}/weather/details?id=1' -H 'Authorization: Bearer ${invalid_token}'" 2>/dev/null || echo "000")

        if [ "$weather_401" = "401" ]; then
            log_info "  /weather/details 无效 Token 返回 401 ✓"
        else
            log_warn "  /weather/details 异常 (HTTP ${weather_401})"
            sleep 2
            continue
        fi

        log_info "  [3/3] 测试 /api/weather (受保护接口)..."
        local api_weather_401
        api_weather_401=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 'http://127.0.0.1:${BACKEND_PORT}/api/weather' -H 'Authorization: Bearer ${invalid_token}'" 2>/dev/null || echo "000")

        if [ "$api_weather_401" = "401" ]; then
            log_info "  /api/weather 无效 Token 返回 401 ✓"
        else
            log_warn "  /api/weather 异常 (HTTP ${api_weather_401})"
            sleep 2
            continue
        fi

        log_info "无效 Token 401 测试通过"
        return 0
    done

    log_error "无效 Token 401 测试失败"
    return 1
}

test_response_content() {
    log_info "  测试 /health 响应内容..."
    local health_resp
    health_resp=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "curl -s --max-time 10 http://127.0.0.1:${BACKEND_PORT}/health" 2>/dev/null || echo "")

    if [[ "$health_resp" =~ "\"status\"" ]] && [[ "$health_resp" =~ "\"database\"" ]]; then
        log_info "  /health JSON 格式正确 ✓"
    else
        log_warn "  /health 响应格式异常: ${health_resp:0:100}"
        return 1
    fi

    log_info "  测试 /health database 字段..."
    if [[ "$health_resp" =~ '"database":"connected"' ]]; then
        log_info "  数据库连接状态正常 ✓"
    else
        log_warn "  数据库连接状态异常"
        return 1
    fi

    log_info "响应内容验证通过"
    return 0
}

rollback_and_exit() {
    local reason="$1"
    log_error "测试失败: ${reason}"
    log_error "自动回滚到上一个正常版本..."

    local last_known_good
    last_known_good=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat ${REMOTE_DIR}/.last_deployed 2>/dev/null || echo """)

    if [ -z "$last_known_good" ]; then
        log_error "无法回滚: 未找到上一个正常版本记录"
        exit 1
    fi

    rollback_on_failure "$last_known_good"
    exit 1
}

# 备份数据库
backup_database() {
    log_step "备份数据库..."

    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'bash -c \"docker exec ${DB_CONTAINER} pg_dump -Fc -U ${DB_USER} ${DB_NAME} > ${REMOTE_DIR}/backups/backup.dump\"'"
        return 0
    fi

    local timestamp
    timestamp=$(date +%Y%m%d_%H%M%S)
    local backup_file="${REMOTE_DIR}/backups/${DB_NAME}_${timestamp}.dump"

    log_info "创建数据库备份..."

    # 创建备份目录
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "mkdir -p ${REMOTE_DIR}/backups"

    # 在远程执行备份（使用 stdout 重定向）
    if ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "bash -c 'docker exec ${DB_CONTAINER} pg_dump -Fc -U ${DB_USER} ${DB_NAME} > /tmp/backup_${timestamp}.dump && mv /tmp/backup_${timestamp}.dump \"${backup_file}\"'" 2>&1; then
        log_info "数据库备份成功: ${DB_NAME}_${timestamp}.dump"

        # 清理旧数据库备份（保留2个）
        cleanup_old_db_backups
    else
        log_warn "数据库备份失败，继续部署"
    fi
}

# 清理旧数据库备份（保留2个）
cleanup_old_db_backups() {
    local keep_count="${1:-1}"

    local backup_count
    backup_count=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "ls -t ${REMOTE_DIR}/backups/${DB_NAME}_*.dump 2>/dev/null | wc -l" || echo "0")

    if [ "$backup_count" -le "$keep_count" ]; then
        return 0
    fi

    local to_delete
    to_delete=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "ls -t ${REMOTE_DIR}/backups/${DB_NAME}_*.dump 2>/dev/null | tail -n +$((keep_count + 1))" || echo "")

    if [ -z "$to_delete" ]; then
        return 0
    fi

    log_step "清理旧数据库备份（保留 ${keep_count} 个）..."
    echo "$to_delete" | while read -r file; do
        if [ -n "$file" ]; then
            ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f '${file}'" 2>/dev/null && echo "  已删除: $(basename ${file})" || true
        fi
    done
}

# 回滚数据库（手动触发）
rollback_database() {
    local backup_file="$1"

    if [ -z "$backup_file" ]; then
        log_error "回滚失败: 未指定备份文件"
        return 1
    fi

    log_step "回滚数据库..."
    log_warn "从备份恢复: ${backup_file}"

    if [ "$DRY_RUN" = true ]; then
        log_dry "pg_restore -c -U ${DB_USER} -d ${DB_NAME} ${backup_file}"
        return 0
    fi

    # 检查备份文件是否存在
    if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" "[ -f '${backup_file}' ]"; then
        log_error "备份文件不存在: ${backup_file}"
        return 1
    fi

    # 停止服务避免数据库冲突
    log_info "停止服务..."
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl stop ${SERVICE_NAME}" 2>/dev/null || true

    # 执行恢复（通过 stdin 管道）
    if ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat '${backup_file}'" | \
        ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "docker exec -i ${DB_CONTAINER} pg_restore -c -U ${DB_USER} -d ${DB_NAME}" 2>&1; then
        log_info "数据库恢复成功"
    else
        log_error "数据库恢复失败"
        return 1
    fi

    # 重启服务
    log_info "重启服务..."
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl start ${SERVICE_NAME}"

    log_info "数据库回滚完成"
}

# 执行数据库迁移
run_migrations() {
    log_step "执行数据库迁移..."

    if [ "$DRY_RUN" = true ]; then
        log_dry "检查并执行 migrations/*.sql"
        local migrations
        migrations=$(ls ./migrations/*.sql 2>/dev/null | wc -l || echo "0")
        log_dry "发现 ${migrations} 个迁移文件"
        return 0
    fi

    # 检查是否有迁移文件
    local sql_files
    sql_files=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls ${REMOTE_DIR}/migrations/*.sql 2>/dev/null | sort" || echo "")

    if [ -n "$sql_files" ]; then
        # 有迁移文件，先备份数据库
        backup_database
    else
        log_info "没有发现迁移文件，跳过"
    fi

    log_info "检查迁移文件..."
    if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" "[ -d ${REMOTE_DIR}/migrations ]"; then
        log_warn "迁移目录不存在，跳过"
        return 0
    fi

    local sql_files
    sql_files=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls ${REMOTE_DIR}/migrations/*.sql 2>/dev/null | sort" || echo "")

    if [ -z "$sql_files" ]; then
        log_warn "没有找到迁移文件"
        return 0
    fi

    for sql_file in $sql_files; do
        local filename
        filename=$(basename "$sql_file")
        local table_check=$(echo "$filename" | sed 's/[0-9]*_//' | sed 's/.sql$//')

        log_info "检查迁移: $filename"

        if [ "$table_check" = "add_refresh_tokens" ]; then
            local exists
            exists=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
                "docker exec ${DB_CONTAINER} psql -U ${DB_USER} -d ${DB_NAME} -tAc \"SELECT 1 FROM information_schema.tables WHERE table_name = 'refresh_tokens'\" 2>/dev/null" || echo "0")
            if [ "$exists" = "1" ]; then
                log_info "  表 refresh_tokens 已存在，跳过"
                continue
            fi
        fi

        log_info "  执行迁移: $filename"
        if ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -i ${DB_CONTAINER} psql -U ${DB_USER} -d ${DB_NAME}" < "$sql_file"; then
            log_info "  完成: $filename"
        else
            log_warn "  迁移可能已执行或失败，继续"
        fi
    done

    log_info "数据库迁移完成"
}

# 确认部署
confirm_deploy() {
    if [ "$DRY_RUN" = true ] || [ "$SKIP_CONFIRM" = true ]; then
        return 0
    fi

    echo ""
    echo -e "${YELLOW}========== 部署确认 ==========${NC}"
    echo -e "  环境:      ${APP_ENV}"
    echo -e "  远程目录:  ${REMOTE_DIR}"
    echo -e "  服务:      ${SERVICE_NAME}"
    echo -e "  Git:       ${GIT_VERSION}"
    echo -e "  测试域名:  ${TEST_DOMAIN}"
    echo -e "================================${NC}"
    echo ""

    read -p "确认部署? [y/N] " -n 1 -r
    echo
    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
        log_warn "部署已取消"
        exit 0
    fi
}

# 显示部署信息
show_deploy_info() {
    echo ""
    echo -e "${BLUE}========================================${NC}"
    echo -e "${BLUE}  Rust Backend 部署脚本${NC}"
    echo -e "${BLUE}========================================${NC}"
    echo -e "  项目:      ${PROJECT_NAME}"
    echo -e "  环境:      ${APP_ENV}"
    echo -e "  远程主机:  ${REMOTE_USER}@${REMOTE_HOST}"
    echo -e "  远程目录:  ${REMOTE_DIR}"
    echo -e "  服务:      ${SERVICE_NAME}"
    echo -e "  Git 版本:  ${GIT_VERSION}"
    if [ "$DRY_RUN" = true ]; then
        echo -e "  模式:      ${YELLOW}预览模式 (DRY-RUN)${NC}"
    fi
    echo -e "${BLUE}========================================${NC}"
    echo ""
}

# 清理函数
cleanup() {
    ssh_disconnect
}

trap cleanup EXIT

# 主函数
main() {
    GIT_VERSION=$(get_git_info)

    if [ "$LIST_BACKUPS" = true ]; then
        ssh_connect
        list_backups
        exit 0
    fi

    if [ "$VIEW_LOGS" = true ]; then
        ssh_connect
        view_logs
        exit 0
    fi

    if [ "$TAIL_LOGS" = true ]; then
        ssh_connect
        tail_logs
        exit 0
    fi

    if [ "$VIEW_STATUS" = true ]; then
        ssh_connect
        status
        exit 0
    fi

    if [ "$ROLLBACK" = true ]; then
        ssh_connect
        rollback
        exit 0
    fi

    show_deploy_info
    confirm_deploy

    check_dependencies
    ssh_connect
    build_project
    backup_old_version
    cleanup_old_backups 5
    upload_binary
    upload_config
    upload_migrations
    upload_test_script
    upload_cleanup_script
    restart_service
    run_migrations

    if run_tests; then
        log_to_file "DEPLOY success"
        send_webhook "success" "部署完成"

        echo ""
        log_info "========== 部署完成！ =========="
        echo -e "  Git:   ${GIT_VERSION}"
        echo -e "  环境:  ${APP_ENV}"
        echo ""

        if [ "$POST_DEPLOY_LOGS" = true ]; then
            echo ""
            log_step "显示最近日志..."
            view_logs
        fi
    else
        log_error "========== 部署失败，已自动回滚 =========="
        echo -e "  Git:   ${GIT_VERSION}"
        echo -e "  环境:  ${APP_ENV}"
        echo ""
        exit 1
    fi
}

main