asd-backend/deploy.sh

#!/bin/bash

# ===========================================
# Rust Backend Deployment Script
# ===========================================
# Usage: ./deploy.sh [development|production] [options]
#
# Options:
#   --dry-run      预览模式，不执行实际操作
#   --yes, -y     跳过确认提示
#   --skip-tests   跳过部署后测试
#   --rollback     回滚到上一个备份版本
#   --backup-list  列出可用备份
# ===========================================

set -euo pipefail

PROJECT_NAME="rust-backend"
REMOTE_USER="root"
REMOTE_HOST="1panel-server"
APP_ENV="production"

# 默认选项
DRY_RUN=false
SKIP_CONFIRM=false
SKIP_TESTS=false
ROLLBACK=false
LIST_BACKUPS=false
VIEW_LOGS=false
TAIL_LOGS=false
LOG_LINES=50
VIEW_STATUS=false
POST_DEPLOY_LOGS=false

# 全局变量（部署流程中使用）
BACKUP_PATH=""

# 解析选项（先处理选项，再处理位置参数）
while [[ $# -gt 0 ]]; do
    case "$1" in
        --dry-run)
            DRY_RUN=true
            ;;
        --yes|-y)
            SKIP_CONFIRM=true
            ;;
        --skip-tests)
            SKIP_TESTS=true
            ;;
        --rollback)
            ROLLBACK=true
            ;;
        --backup-list)
            LIST_BACKUPS=true
            ;;
        --logs)
            VIEW_LOGS=true
            ;;
        --tail)
            TAIL_LOGS=true
            ;;
        --logs=*)
            VIEW_LOGS=true
            LOG_LINES="${1#*=}"
            ;;
        --tail=*)
            TAIL_LOGS=true
            LOG_LINES="${1#*=}"
            ;;
        --status)
            VIEW_STATUS=true
            ;;
        --post-logs)
            POST_DEPLOY_LOGS=true
            ;;
        --help|-h)
            echo "用法: $0 [development|production] [options]"
            echo ""
            echo "选项:"
            echo "  --dry-run      预览模式，不执行实际操作"
            echo "  --yes, -y      跳过确认提示"
            echo "  --skip-tests   跳过部署后测试"
            echo "  --rollback     回滚到上一个备份版本"
            echo "  --backup-list  列出可用备份"
            echo "  --logs [N]     查看后端日志（默认50行）"
            echo "  --tail [N]    实时跟踪后端日志（Ctrl+C 退出）"
            echo "  --status      查看服务状态"
            echo "  --post-logs    部署后自动显示日志"
            exit 0
            ;;
        *)
            # 位置参数：development 或 production
            if [[ "$1" == "development" || "$1" == "production" ]]; then
                APP_ENV="$1"
            else
                echo "未知选项: $1"
                echo "用法: $0 [development|production] [options]"
                exit 1
            fi
            ;;
    esac
    shift
done

# 根据环境选择远程目录和服务
case "${APP_ENV}" in
    development)
        REMOTE_DIR="/root/rust/rust_backend_dev"
        SERVICE_NAME="rust-backend-dev.service"
        TEST_DOMAIN="https://dev.xmclassmate.top"
        BACKEND_PORT="8080"
        DB_CONTAINER="1Panel-postgresql-FtMo"
        DB_NAME="milkydata_dev"
        DB_USER="milkydata"
        ;;
    production)
        REMOTE_DIR="/root/rust/rust_backend"
        SERVICE_NAME="rust-backend.service"
        TEST_DOMAIN="https://xmclassmate.top"
        BACKEND_PORT="4433"
        DB_CONTAINER="1Panel-postgresql-FtMo"
        DB_NAME="milkydata"
        DB_USER="milkydata"
        ;;
    *)
        echo "无效的环境: ${APP_ENV}"
        echo "支持的環境: development, production"
        exit 1
        ;;
esac

# 彩色输出
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1" >&2; }
log_step() { echo -e "${BLUE}[STEP]${NC} $1"; }
log_dry() { echo -e "${YELLOW}[DRY-RUN]${NC} $1"; }

# 获取 Git 版本信息
get_git_info() {
    local git_branch
    local git_commit
    local git_version

    git_branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
    git_commit=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
    git_version="${git_branch}@${git_commit}"

    echo "$git_version"
}

# 部署日志路径
get_deploy_log() {
    echo "${REMOTE_DIR}/deploy.log"
}

# 记录部署日志
log_to_file() {
    if [ "$DRY_RUN" = true ]; then
        return 0
    fi

    local timestamp
    timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    local log_entry="[${timestamp}] ${APP_ENV} ${GIT_VERSION} $1"

    ssh "${REMOTE_USER}@${REMOTE_HOST}" "echo '${log_entry}' >> $(get_deploy_log)" 2>/dev/null || true
}

# Webhook 通知
send_webhook() {
    local status="$1"
    local message="$2"

    if [ -z "${WEBHOOK_URL:-}" ]; then
        return 0
    fi

    local payload
    payload=$(cat <<EOF
{
    "app": "rust-backend",
    "env": "${APP_ENV}",
    "status": "${status}",
    "version": "${GIT_VERSION}",
    "message": "${message}",
    "timestamp": "$(date -Iseconds)"
}
EOF
)

    if [ "$DRY_RUN" = true ]; then
        log_dry "curl -X POST '${WEBHOOK_URL}' -d '${payload}'"
        return 0
    fi

    curl -s -X POST "${WEBHOOK_URL}" \
        -H "Content-Type: application/json" \
        -d "${payload}" >/dev/null 2>&1 || true
}

# 列出备份
list_backups() {
    log_step "可用备份列表..."
    echo ""

    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'ls -la ${REMOTE_DIR}/${PROJECT_NAME}.backup.*'"
        return 0
    fi

    ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls -la ${REMOTE_DIR}/${PROJECT_NAME}.backup.*" 2>/dev/null || echo "无备份文件"
    echo ""
}

# 查看后端日志
view_logs() {
    local lines="${LOG_LINES:-50}"
    log_step "查看后端日志（最后 ${lines} 行）..."
    echo ""

    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'tail -${lines} ${REMOTE_DIR}/logs/rust-backend.*.log'"
        return 0
    fi

    # 获取最新的日志文件
    local log_file
    log_file=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls -t ${REMOTE_DIR}/logs/rust-backend.*.log 2>/dev/null | head -1" || echo "")

    if [ -z "$log_file" ]; then
        log_warn "未找到日志文件: ${REMOTE_DIR}/logs/"
        return 1
    fi

    echo -e "${BLUE}=== ${log_file} ===${NC}"
    echo ""
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "tail -${lines} \"${log_file}\""
    echo ""
}

# 实时跟踪后端日志
tail_logs() {
    local lines="${LOG_LINES:-100}"
    log_step "实时跟踪后端日志（最后 ${lines} 行，按 Ctrl+C 退出）..."
    echo ""

    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'tail -f -n ${lines} ${REMOTE_DIR}/logs/rust-backend.*.log'"
        return 0
    fi

    # 获取最新的日志文件
    local log_file
    log_file=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls -t ${REMOTE_DIR}/logs/rust-backend.*.log 2>/dev/null | head -1" || echo "")

    if [ -z "$log_file" ]; then
        log_warn "未找到日志文件: ${REMOTE_DIR}/logs/"
        return 1
    fi

    echo -e "${BLUE}=== 实时日志: ${log_file} ===${NC}"
    echo -e "${YELLOW}按 Ctrl+C 退出${NC}"
    echo ""
    ssh -t "${REMOTE_USER}@${REMOTE_HOST}" "tail -f -n ${lines} \"${log_file}\""
}

# 查看服务状态
status() {
    log_step "服务状态..."
    echo ""

    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'systemctl status ${SERVICE_NAME}'"
        return 0
    fi

    echo -e "${BLUE}=== ${SERVICE_NAME} ===${NC}"
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl status ${SERVICE_NAME} --no-pager"
    echo ""

    echo -e "${BLUE}=== 最近日志 ===${NC}"
    local log_file
    log_file=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls -t ${REMOTE_DIR}/logs/rust-backend.*.log 2>/dev/null | head -1" || echo "")
    if [ -n "$log_file" ]; then
        echo -e "${BLUE}=== ${log_file} ===${NC}"
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "tail -20 \"${log_file}\""
    fi
    echo ""
}

# 回滚（手动触发）
rollback() {
    log_step "查找最新备份..."
    local latest_backup

    if [ "$DRY_RUN" = true ]; then
        latest_backup="${REMOTE_DIR}/${PROJECT_NAME}.backup.20240101_120000"
        log_dry "latest_backup=${latest_backup}"
    else
        latest_backup=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "ls -t ${REMOTE_DIR}/${PROJECT_NAME}.backup.* 2>/dev/null | head -1" || echo "")

        if [ -z "$latest_backup" ]; then
            log_error "没有找到可用的备份"
            return 1
        fi
    fi

    echo ""
    echo -e "${YELLOW}========== 回滚确认 ==========${NC}"
    echo -e "  将回滚到: ${latest_backup}"
    echo -e "  服务: ${SERVICE_NAME}"
    echo -e "================================${NC}"
    echo ""

    if [ "$SKIP_CONFIRM" = false ]; then
        read -p "确认回滚? [y/N] " -n 1 -r
        echo
        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
            log_warn "回滚已取消"
            return 0
        fi
    fi

    log_step "停止服务..."
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl stop ${SERVICE_NAME}" 2>/dev/null || log_warn "停止服务失败，继续尝试..."

    log_step "恢复备份..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'cp ${latest_backup} ${REMOTE_DIR}/${PROJECT_NAME}'"
    else
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "cp ${latest_backup} ${REMOTE_DIR}/${PROJECT_NAME}"
    fi

    log_step "启动服务..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'systemctl start ${SERVICE_NAME}'"
    else
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl start ${SERVICE_NAME}"
    fi

    log_info "回滚完成"
    log_to_file "MANUAL_ROLLBACK to $(basename $latest_backup)"
    send_webhook "rollback" "手动回滚完成: $(basename $latest_backup)"
}

# 回滚到上一个正常版本（测试失败时自动触发）
rollback_on_failure() {
    local backup_path="$1"

    if [ -z "$backup_path" ]; then
        log_error "回滚失败: 未指定备份路径"
        return 1
    fi

    echo ""
    log_error "========== 测试失败，自动回滚 =========="
    echo -e "  原因: 部署后测试未通过"
    echo -e "  回滚到: ${backup_path}"
    echo -e "  服务: ${SERVICE_NAME}"
    echo -e "=========================================="
    echo ""

    log_step "停止服务..."
    if [ "$DRY_RUN" = false ]; then
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl stop ${SERVICE_NAME}" 2>/dev/null || log_warn "停止服务失败，继续尝试..."
    else
        log_dry "systemctl stop ${SERVICE_NAME}"
    fi

    log_step "恢复上一个正常版本..."
    if [ "$DRY_RUN" = false ]; then
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "cp '${backup_path}' '${REMOTE_DIR}/${PROJECT_NAME}'"
    else
        log_dry "cp '${backup_path}' '${REMOTE_DIR}/${PROJECT_NAME}'"
    fi

    log_step "启动服务..."
    if [ "$DRY_RUN" = false ]; then
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl start ${SERVICE_NAME}"
        sleep 3
    else
        log_dry "systemctl start ${SERVICE_NAME}"
    fi

    log_info "自动回滚完成"
    log_to_file "AUTO_ROLLBACK to $(basename $backup_path) - tests failed"
    send_webhook "failure" "部署测试失败，已自动回滚到: $(basename $backup_path)"
}

# 清理旧备份，只保留最近 N 个
cleanup_old_backups() {
    local keep_count="${1:-5}"

    if [ "$DRY_RUN" = true ]; then
        log_dry "保留最近 ${keep_count} 个备份，删除其余"
        return 0
    fi

    local backup_count
    backup_count=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "ls -t ${REMOTE_DIR}/${PROJECT_NAME}.backup.* 2>/dev/null | wc -l" || echo "0")

    if [ "$backup_count" -le "$keep_count" ]; then
        return 0
    fi

    local to_delete
    to_delete=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "ls -t ${REMOTE_DIR}/${PROJECT_NAME}.backup.* 2>/dev/null | tail -n +$((keep_count + 1))" || echo "")

    if [ -z "$to_delete" ]; then
        return 0
    fi

    log_step "清理旧备份（保留 ${keep_count} 个）..."
    echo "$to_delete" | while read -r file; do
        if [ -n "$file" ]; then
            ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f '${file}'" 2>/dev/null && echo "  已删除: ${file}" || true
        fi
    done
}

# 检查依赖
check_dependencies() {
    log_step "检查依赖..."
    local deps=("cargo" "rsync" "ssh")
    for cmd in "${deps[@]}"; do
        if ! command -v "$cmd" &> /dev/null; then
            log_error "依赖命令 '$cmd' 未找到，请先安装。"
        fi
    done
    log_info "依赖检查通过"
}

# 获取远程服务器 SSH ControlMaster socket 路径
get_ssh_socket() {
    echo "/tmp/ssh-master-${REMOTE_HOST}-$$"
}

# 建立 SSH 连接（复用）
ssh_connect() {
    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh -o ControlMaster=auto -o ControlPath=$(get_ssh_socket) -o ControlPersist=5 ${REMOTE_USER}@${REMOTE_HOST}"
        return 0
    fi

    local socket
    socket=$(get_ssh_socket)

    # 检查是否已有连接
    if ! ssh -o "ControlPath=${socket}" -o "ConnectionAttempts=3" -o "ConnectTimeout=10" \
        -o "ControlMaster=no" -o "ControlPersist=no" \
        "${REMOTE_USER}@${REMOTE_HOST}" "echo 'connection test'" &>/dev/null; then

        log_info "建立 SSH 连接..."
        ssh -o ControlMaster=auto -o ControlPath="${socket}" -o ControlPersist=600 \
            -o ConnectionAttempts=3 -o ConnectTimeout=10 \
            -N "${REMOTE_USER}@${REMOTE_HOST}" &
        sleep 2
    fi
}

# 关闭 SSH 连接
ssh_disconnect() {
    if [ "$DRY_RUN" = true ]; then
        return 0
    fi

    local socket
    socket=$(get_ssh_socket)
    ssh -o "ControlPath=${socket}" -O exit "${REMOTE_USER}@${REMOTE_HOST}" 2>/dev/null || true
}

# 编译项目
build_project() {
    log_step "编译项目 (release模式)..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "APP_ENV=\"${APP_ENV}\" cargo build --release"
        return 0
    fi

    if APP_ENV="${APP_ENV}" cargo build --release; then
        log_info "编译成功"
    else
        log_error "编译失败！"
    fi
}

# 备份旧版本并保存回滚标记
backup_old_version() {
    log_step "备份旧版本..."
    local binary_path="./target/release/${PROJECT_NAME}"
    local timestamp
    timestamp=$(date +%Y%m%d_%H%M%S)
    BACKUP_PATH="${REMOTE_DIR}/${PROJECT_NAME}.backup.${timestamp}"

    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'if [ -f ${REMOTE_DIR}/${PROJECT_NAME} ]; then cp ${REMOTE_DIR}/${PROJECT_NAME} ${BACKUP_PATH}; echo \"\${BACKUP_PATH}\" > ${REMOTE_DIR}/.last_deployed; fi'"
        return 0
    fi

    ssh "${REMOTE_USER}@${REMOTE_HOST}" "
        if [ -f '${REMOTE_DIR}/${PROJECT_NAME}' ]; then
            cp '${REMOTE_DIR}/${PROJECT_NAME}' '${BACKUP_PATH}'
            echo '${BACKUP_PATH}' > '${REMOTE_DIR}/.last_deployed'
            echo '已备份到: ${BACKUP_PATH}'
        else
            echo '无旧版本需要备份'
        fi
    "
    log_info "备份完成"
}

# 上传二进制文件
upload_binary() {
    log_step "上传二进制文件..."
    local binary_path="./target/release/${PROJECT_NAME}"

    if [ ! -f "$binary_path" ]; then
        log_error "编译产物未找到: $binary_path"
    fi

    if [ "$DRY_RUN" = true ]; then
        log_dry "rsync -avzP '$binary_path' '${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/'"
        return 0
    fi

    rsync -avzP --progress "$binary_path" "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/"
    log_info "上传完成"
}

# 上传配置文件（只上传 .toml 文件）
upload_config() {
    log_step "上传配置文件..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "rsync -avzP --include='*.toml' --exclude='*' ./config/ '${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/config/'"
        return 0
    fi

    rsync -avzP --include='*.toml' --exclude='*' ./config/ "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/config/"
    log_info "配置文件上传完成"
}

# 上传迁移文件
upload_migrations() {
    log_step "上传迁移文件..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "rsync -avzP ./migrations/ '${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/migrations/'"
        return 0
    fi

    if [ ! -d "./migrations" ]; then
        log_warn "迁移目录不存在，跳过"
        return 0
    fi

    rsync -avzP --progress ./migrations/ "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/migrations/"
    log_info "迁移文件上传完成"
}

# 上传测试脚本
upload_test_script() {
    log_step "上传测试脚本..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "rsync -avzP ./test_deployment.sh '${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/'"
        return 0
    fi

    rsync -avzP --progress ./test_deployment.sh "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/"
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "chmod +x ${REMOTE_DIR}/test_deployment.sh"
    log_info "测试脚本上传完成"
}

# 上传清理脚本（定时任务用）
upload_cleanup_script() {
    log_step "上传清理脚本..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "rsync -avzP ./scripts/cleanup_refresh_tokens.sh '${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/scripts/'"
        return 0
    fi

    mkdir -p ./scripts 2>/dev/null || true
    if [ -f "./scripts/cleanup_refresh_tokens.sh" ]; then
        rsync -avzP --progress ./scripts/cleanup_refresh_tokens.sh "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DIR}/scripts/"
        ssh "${REMOTE_USER}@${REMOTE_HOST}" "chmod +x ${REMOTE_DIR}/scripts/cleanup_refresh_tokens.sh"
        log_info "清理脚本上传完成"
    else
        log_warn "清理脚本不存在，跳过"
    fi
}

# 重启远程服务
restart_service() {
    log_step "重启服务 '${SERVICE_NAME}'..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'systemctl restart ${SERVICE_NAME}'"
        return 0
    fi

    if ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl restart ${SERVICE_NAME}"; then
        log_info "服务重启成功"
    else
        log_error "服务重启失败！请检查服务状态和日志。"
    fi

    # 等待服务启动
    sleep 3
}

run_tests() {
    log_step "执行部署后测试..."
    if [ "$DRY_RUN" = true ]; then
        log_dry "步骤1: curl http://127.0.0.1:${BACKEND_PORT}/health"
        log_dry "步骤2: curl ${TEST_DOMAIN}/health"
        return 0
    fi

    if [ "$SKIP_TESTS" = true ]; then
        log_warn "跳过测试（--skip-tests）"
        return 0
    fi

    log_step "步骤1: 测试本地后端..."
    if ! test_local_backend; then
        rollback_and_exit "本地后端测试失败"
    fi

    log_step "步骤2: 测试域名访问..."
    if ! test_domain_access; then
        rollback_and_exit "域名访问测试失败"
    fi

    log_info "测试通过"
    return 0
}

test_local_backend() {
    local max_retries=3
    local retry=0

    while [ $retry -lt $max_retries ]; do
        retry=$((retry + 1))
        log_info "尝试 ${retry}/${max_retries}: 测试本地后端 (127.0.0.1:${BACKEND_PORT})"

        # 测试 1: 健康检查
        log_info "  [1/4] 测试 /health..."
        local health_code
        health_code=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 http://127.0.0.1:${BACKEND_PORT}/health" 2>/dev/null || echo "000")

        if [ "$health_code" != "200" ]; then
            log_warn "  /health 异常 (HTTP ${health_code})"
            sleep 2
            continue
        fi
        log_info "  /health 正常"

        # 测试 2: 登录接口存在（预期 400，微信 code 无效）
        log_info "  [2/4] 测试 /api/login..."
        local login_code
        login_code=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 -X POST http://127.0.0.1:${BACKEND_PORT}/api/login -H 'Content-Type: application/json' -d '{\"code\":\"test_code\"'" 2>/dev/null || echo "000")

        if [ "$login_code" = "400" ]; then
            log_info "  /api/login 正常 (HTTP ${login_code}, 微信返回 400 expected)"
        elif [ "$login_code" = "200" ]; then
            log_info "  /api/login 正常 (HTTP ${login_code})"
        else
            log_warn "  /api/login 异常 (HTTP ${login_code})"
            sleep 2
            continue
        fi

        # 测试 3: Refresh Token 接口存在
        log_info "  [3/4] 测试 /api/refresh-token..."
        local refresh_code
        refresh_code=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 -X POST http://127.0.0.1:${BACKEND_PORT}/api/refresh-token -H 'Content-Type: application/json' -d '{\"refresh_token\":\"invalid\"}'" 2>/dev/null || echo "000")

        if [ "$refresh_code" = "401" ]; then
            log_info "  /api/refresh-token 正常 (HTTP ${refresh_code}, 无效 token expected)"
        elif [ "$refresh_code" = "200" ]; then
            log_info "  /api/refresh-token 正常 (HTTP ${refresh_code})"
        else
            log_warn "  /api/refresh-token 异常 (HTTP ${refresh_code})"
            sleep 2
            continue
        fi

        # 测试 4: 根路径
        log_info "  [4/4] 测试 / (根路径)..."
        local root_code
        root_code=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 http://127.0.0.1:${BACKEND_PORT}/" 2>/dev/null || echo "000")

        if [ "$root_code" = "200" ] || [ "$root_code" = "404" ]; then
            log_info "  / 正常 (HTTP ${root_code})"
        else
            log_warn "  / 异常 (HTTP ${root_code})"
            sleep 2
            continue
        fi

        log_info "本地后端所有测试通过"
        return 0
    done

    log_error "本地后端测试失败"
    return 1
}

test_domain_access() {
    local max_retries=3
    local retry=0

    while [ $retry -lt $max_retries ]; do
        retry=$((retry + 1))
        log_info "尝试 ${retry}/${max_retries}: 测试域名访问 (${TEST_DOMAIN})"

        # 测试 1: 健康检查
        log_info "  [1/4] 测试 /health..."
        local health_code
        health_code=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 -k ${TEST_DOMAIN}/health" 2>/dev/null || echo "000")

        if [ "$health_code" != "200" ]; then
            log_warn "  /health 异常 (HTTP ${health_code})"
            sleep 2
            continue
        fi
        log_info "  /health 正常"

        # 测试 2: 登录接口
        log_info "  [2/4] 测试 /api/login..."
        local login_code
        login_code=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 -k -X POST ${TEST_DOMAIN}/api/login -H 'Content-Type: application/json' -d '{\"code\":\"test\"'" 2>/dev/null || echo "000")

        if [ "$login_code" = "400" ] || [ "$login_code" = "200" ]; then
            log_info "  /api/login 正常 (HTTP ${login_code})"
        else
            log_warn "  /api/login 异常 (HTTP ${login_code})"
            sleep 2
            continue
        fi

        # 测试 3: Refresh Token 接口
        log_info "  [3/4] 测试 /api/refresh-token..."
        local refresh_code
        refresh_code=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 -k -X POST ${TEST_DOMAIN}/api/refresh-token -H 'Content-Type: application/json' -d '{\"refresh_token\":\"invalid\"}'" 2>/dev/null || echo "000")

        if [ "$refresh_code" = "401" ] || [ "$refresh_code" = "200" ]; then
            log_info "  /api/refresh-token 正常 (HTTP ${refresh_code})"
        else
            log_warn "  /api/refresh-token 异常 (HTTP ${refresh_code})"
            sleep 2
            continue
        fi

        # 测试 4: 根路径
        log_info "  [4/4] 测试 / (根路径)..."
        local root_code
        root_code=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
            "curl -s -o /dev/null -w '%{http_code}' --max-time 10 -k ${TEST_DOMAIN}/" 2>/dev/null || echo "000")

        if [ "$root_code" = "200" ] || [ "$root_code" = "404" ]; then
            log_info "  / 正常 (HTTP ${root_code})"
        else
            log_warn "  / 异常 (HTTP ${root_code})"
            sleep 2
            continue
        fi

        log_info "域名访问所有测试通过"
        return 0
    done

    log_error "域名访问测试失败"
    return 1
}

rollback_and_exit() {
    local reason="$1"
    log_error "测试失败: ${reason}"
    log_error "自动回滚到上一个正常版本..."

    local last_known_good
    last_known_good=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat ${REMOTE_DIR}/.last_deployed 2>/dev/null || echo """)

    if [ -z "$last_known_good" ]; then
        log_error "无法回滚: 未找到上一个正常版本记录"
        exit 1
    fi

    rollback_on_failure "$last_known_good"
    exit 1
}

# 备份数据库
backup_database() {
    log_step "备份数据库..."

    if [ "$DRY_RUN" = true ]; then
        log_dry "ssh ${REMOTE_USER}@${REMOTE_HOST} 'bash -c \"docker exec ${DB_CONTAINER} pg_dump -Fc -U ${DB_USER} ${DB_NAME} > ${REMOTE_DIR}/backups/backup.dump\"'"
        return 0
    fi

    local timestamp
    timestamp=$(date +%Y%m%d_%H%M%S)
    local backup_file="${REMOTE_DIR}/backups/${DB_NAME}_${timestamp}.dump"

    log_info "创建数据库备份..."

    # 创建备份目录
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "mkdir -p ${REMOTE_DIR}/backups"

    # 在远程执行备份（使用 stdout 重定向）
    if ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "bash -c 'docker exec ${DB_CONTAINER} pg_dump -Fc -U ${DB_USER} ${DB_NAME} > /tmp/backup_${timestamp}.dump && mv /tmp/backup_${timestamp}.dump \"${backup_file}\"'" 2>&1; then
        log_info "数据库备份成功: ${DB_NAME}_${timestamp}.dump"

        # 清理旧数据库备份（保留2个）
        cleanup_old_db_backups
    else
        log_warn "数据库备份失败，继续部署"
    fi
}

# 清理旧数据库备份（保留2个）
cleanup_old_db_backups() {
    local keep_count="${1:-1}"

    local backup_count
    backup_count=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "ls -t ${REMOTE_DIR}/backups/${DB_NAME}_*.dump 2>/dev/null | wc -l" || echo "0")

    if [ "$backup_count" -le "$keep_count" ]; then
        return 0
    fi

    local to_delete
    to_delete=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "ls -t ${REMOTE_DIR}/backups/${DB_NAME}_*.dump 2>/dev/null | tail -n +$((keep_count + 1))" || echo "")

    if [ -z "$to_delete" ]; then
        return 0
    fi

    log_step "清理旧数据库备份（保留 ${keep_count} 个）..."
    echo "$to_delete" | while read -r file; do
        if [ -n "$file" ]; then
            ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f '${file}'" 2>/dev/null && echo "  已删除: $(basename ${file})" || true
        fi
    done
}

# 回滚数据库（手动触发）
rollback_database() {
    local backup_file="$1"

    if [ -z "$backup_file" ]; then
        log_error "回滚失败: 未指定备份文件"
        return 1
    fi

    log_step "回滚数据库..."
    log_warn "从备份恢复: ${backup_file}"

    if [ "$DRY_RUN" = true ]; then
        log_dry "pg_restore -c -U ${DB_USER} -d ${DB_NAME} ${backup_file}"
        return 0
    fi

    # 检查备份文件是否存在
    if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" "[ -f '${backup_file}' ]"; then
        log_error "备份文件不存在: ${backup_file}"
        return 1
    fi

    # 停止服务避免数据库冲突
    log_info "停止服务..."
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl stop ${SERVICE_NAME}" 2>/dev/null || true

    # 执行恢复（通过 stdin 管道）
    if ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat '${backup_file}'" | \
        ssh "${REMOTE_USER}@${REMOTE_HOST}" \
        "docker exec -i ${DB_CONTAINER} pg_restore -c -U ${DB_USER} -d ${DB_NAME}" 2>&1; then
        log_info "数据库恢复成功"
    else
        log_error "数据库恢复失败"
        return 1
    fi

    # 重启服务
    log_info "重启服务..."
    ssh "${REMOTE_USER}@${REMOTE_HOST}" "systemctl start ${SERVICE_NAME}"

    log_info "数据库回滚完成"
}

# 执行数据库迁移
run_migrations() {
    log_step "执行数据库迁移..."

    if [ "$DRY_RUN" = true ]; then
        log_dry "检查并执行 migrations/*.sql"
        local migrations
        migrations=$(ls ./migrations/*.sql 2>/dev/null | wc -l || echo "0")
        log_dry "发现 ${migrations} 个迁移文件"
        return 0
    fi

    # 检查是否有迁移文件
    local sql_files
    sql_files=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls ${REMOTE_DIR}/migrations/*.sql 2>/dev/null | sort" || echo "")

    if [ -n "$sql_files" ]; then
        # 有迁移文件，先备份数据库
        backup_database
    else
        log_info "没有发现迁移文件，跳过"
    fi

    log_info "检查迁移文件..."
    if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" "[ -d ${REMOTE_DIR}/migrations ]"; then
        log_warn "迁移目录不存在，跳过"
        return 0
    fi

    local sql_files
    sql_files=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "ls ${REMOTE_DIR}/migrations/*.sql 2>/dev/null | sort" || echo "")

    if [ -z "$sql_files" ]; then
        log_warn "没有找到迁移文件"
        return 0
    fi

    for sql_file in $sql_files; do
        local filename
        filename=$(basename "$sql_file")
        local table_check=$(echo "$filename" | sed 's/[0-9]*_//' | sed 's/.sql$//')

        log_info "检查迁移: $filename"

        if [ "$table_check" = "add_refresh_tokens" ]; then
            local exists
            exists=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" \
                "docker exec ${DB_CONTAINER} psql -U ${DB_USER} -d ${DB_NAME} -tAc \"SELECT 1 FROM information_schema.tables WHERE table_name = 'refresh_tokens'\" 2>/dev/null" || echo "0")
            if [ "$exists" = "1" ]; then
                log_info "  表 refresh_tokens 已存在，跳过"
                continue
            fi
        fi

        log_info "  执行迁移: $filename"
        if ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -i ${DB_CONTAINER} psql -U ${DB_USER} -d ${DB_NAME}" < "$sql_file"; then
            log_info "  完成: $filename"
        else
            log_warn "  迁移可能已执行或失败，继续"
        fi
    done

    log_info "数据库迁移完成"
}

# 确认部署
confirm_deploy() {
    if [ "$DRY_RUN" = true ] || [ "$SKIP_CONFIRM" = true ]; then
        return 0
    fi

    echo ""
    echo -e "${YELLOW}========== 部署确认 ==========${NC}"
    echo -e "  环境:      ${APP_ENV}"
    echo -e "  远程目录:  ${REMOTE_DIR}"
    echo -e "  服务:      ${SERVICE_NAME}"
    echo -e "  Git:       ${GIT_VERSION}"
    echo -e "  测试域名:  ${TEST_DOMAIN}"
    echo -e "================================${NC}"
    echo ""

    read -p "确认部署? [y/N] " -n 1 -r
    echo
    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
        log_warn "部署已取消"
        exit 0
    fi
}

# 显示部署信息
show_deploy_info() {
    echo ""
    echo -e "${BLUE}========================================${NC}"
    echo -e "${BLUE}  Rust Backend 部署脚本${NC}"
    echo -e "${BLUE}========================================${NC}"
    echo -e "  项目:      ${PROJECT_NAME}"
    echo -e "  环境:      ${APP_ENV}"
    echo -e "  远程主机:  ${REMOTE_USER}@${REMOTE_HOST}"
    echo -e "  远程目录:  ${REMOTE_DIR}"
    echo -e "  服务:      ${SERVICE_NAME}"
    echo -e "  Git 版本:  ${GIT_VERSION}"
    if [ "$DRY_RUN" = true ]; then
        echo -e "  模式:      ${YELLOW}预览模式 (DRY-RUN)${NC}"
    fi
    echo -e "${BLUE}========================================${NC}"
    echo ""
}

# 清理函数
cleanup() {
    ssh_disconnect
}

trap cleanup EXIT

# 主函数
main() {
    GIT_VERSION=$(get_git_info)

    if [ "$LIST_BACKUPS" = true ]; then
        ssh_connect
        list_backups
        exit 0
    fi

    if [ "$VIEW_LOGS" = true ]; then
        ssh_connect
        view_logs
        exit 0
    fi

    if [ "$TAIL_LOGS" = true ]; then
        ssh_connect
        tail_logs
        exit 0
    fi

    if [ "$VIEW_STATUS" = true ]; then
        ssh_connect
        status
        exit 0
    fi

    if [ "$ROLLBACK" = true ]; then
        ssh_connect
        rollback
        exit 0
    fi

    show_deploy_info
    confirm_deploy

    check_dependencies
    ssh_connect
    build_project
    backup_old_version
    cleanup_old_backups 5
    upload_binary
    upload_config
    upload_migrations
    upload_test_script
    upload_cleanup_script
    restart_service
    run_migrations

    if run_tests; then
        log_to_file "DEPLOY success"
        send_webhook "success" "部署完成"

        echo ""
        log_info "========== 部署完成！ =========="
        echo -e "  Git:   ${GIT_VERSION}"
        echo -e "  环境:  ${APP_ENV}"
        echo ""

        if [ "$POST_DEPLOY_LOGS" = true ]; then
            echo ""
            log_step "显示最近日志..."
            view_logs
        fi
    else
        log_error "========== 部署失败，已自动回滚 =========="
        echo -e "  Git:   ${GIT_VERSION}"
        echo -e "  环境:  ${APP_ENV}"
        echo ""
        exit 1
    fi
}

main