#!/bin/bash # ============================================ # 余额同步端到端测试脚本 # 仿照真实用户使用场景测试 Master-Slave 余额同步 # # 用法: ./test_region_sync_e2e.sh # ============================================ set -euo pipefail # ==================== 配置 ==================== MASTER="http://172.22.57.85:23000" SLAVE="http://172.22.57.85:13000" SYNC_API_KEY="123456789" MASTER_USER_KEY="sk-LDEPGBf5QJYgralE0cIJ47NGsaN8odTqn3fs5IGGJtOdLKSE:1" SLAVE_USER_KEY="sk-5zVnpGLzKxN4xeiWTBh8nuIdXajhEcBPlIZN29L0gpLdAwsu:1" USER_ID=7 MODEL="deepseek-ai/DeepSeek-V3-0324" SYNC_WAIT_SECONDS=70 # 等待同步的时间(秒),需大于 SyncIntervalSeconds SYNC_POLL_INTERVAL=5 # 轮询间隔(秒) CONCURRENT_COUNT=5 # 并发请求数量 DEDUCT_AMOUNT=500 # 直接同步 API 测试的扣费金额 TIMEOUT=60 # curl 超时时间(秒) # ==================== 颜色定义 ==================== GREEN='\033[0;32m' RED='\033[0;31m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' BOLD='\033[1m' NC='\033[0m' # ==================== 计数器 ==================== PASS_COUNT=0 FAIL_COUNT=0 SKIP_COUNT=0 TOTAL_TESTS=0 # ==================== 工具函数 ==================== log_pass() { echo -e " ${GREEN}[PASS]${NC} $1" ((PASS_COUNT++)) || true ((TOTAL_TESTS++)) || true } log_fail() { echo -e " ${RED}[FAIL]${NC} $1" ((FAIL_COUNT++)) || true ((TOTAL_TESTS++)) || true } log_skip() { echo -e " ${YELLOW}[SKIP]${NC} $1" ((SKIP_COUNT++)) || true ((TOTAL_TESTS++)) || true } log_info() { echo -e " ${BLUE}[INFO]${NC} $1" } log_warn() { echo -e " ${YELLOW}[WARN]${NC} $1" } print_header() { echo "" echo -e "${CYAN}============================================${NC}" echo -e "${BOLD}$1${NC}" echo -e "${CYAN}============================================${NC}" echo "" } print_section() { echo "" echo -e "${BOLD}--- $1 ---${NC}" } # 查询 Master 余额 query_master_balance() { local user_id=${1:-$USER_ID} local result=$(curl -s -m $TIMEOUT -X POST "$MASTER/api/internal/sync/quota/query" \ -H "Content-Type: application/json" \ -H "X-Sync-API-Key: $SYNC_API_KEY" \ -H "X-Sync-Node: test-node" \ -d "{\"user_id\": $user_id}") echo "$result" | grep -o '"quota":[0-9]*' | grep -o '[0-9]*' } # 等待同步完成(轮询 Master 余额,直到变化或超时) wait_for_sync() { local balance_before=$1 local description=$2 local waited=0 echo -e " ${BLUE}[SYNC]${NC} 等待余额同步到 Master($description)..." >&2 while [ $waited -lt $SYNC_WAIT_SECONDS ]; do local current=$(query_master_balance) if [ -n "$current" ] && [ "$current" != "$balance_before" ]; then echo -e " ${GREEN}[SYNC]${NC} 检测到余额变化(等待了 ${waited}s):$balance_before → $current" >&2 # 只向 stdout 输出纯数字 echo "$current" return 0 fi sleep $SYNC_POLL_INTERVAL waited=$((waited + SYNC_POLL_INTERVAL)) echo -e " ${BLUE}[SYNC]${NC} 已等待 ${waited}s / ${SYNC_WAIT_SECONDS}s" >&2 done # 超时后再查一次 local final=$(query_master_balance) echo -e " ${YELLOW}[SYNC]${NC} 等待超时(${SYNC_WAIT_SECONDS}s),最终余额: $final" >&2 echo "${final:-$balance_before}" } # 发送聊天请求 send_chat() { local stream=${1:-false} local max_tokens=${2:-10} local prompt=${3:-"Say hello in one word"} curl -s -m $TIMEOUT --no-buffer -X POST "$SLAVE/v1/chat/completions" \ -H "Authorization: Bearer $SLAVE_USER_KEY" \ -H "Content-Type: application/json" \ -d "{ \"model\": \"$MODEL\", \"messages\": [{\"role\": \"user\", \"content\": \"$prompt\"}], \"max_tokens\": $max_tokens, \"stream\": $stream }" } # ==================== 脚本开始 ==================== print_header "余额同步端到端测试 (真实用户场景)" echo -e " Master: $MASTER" echo -e " Slave: $SLAVE" echo -e " 测试模型: $MODEL" echo -e " 用户 ID: $USER_ID" echo -e " 同步等待: ${SYNC_WAIT_SECONDS}s" echo -e " 并发数量: $CONCURRENT_COUNT" # ==================== 阶段 0: 环境验证 ==================== print_header "阶段 0: 环境验证" # T0.1: Master 连通性 print_section "T0.1 Master 同步 API 连通性" MASTER_RESULT=$(curl -s -m $TIMEOUT -w "\n%{http_code}" -X POST "$MASTER/api/internal/sync/quota/query" \ -H "Content-Type: application/json" \ -H "X-Sync-API-Key: $SYNC_API_KEY" \ -H "X-Sync-Node: test-node" \ -d "{\"user_id\": $USER_ID}") MASTER_HTTP_CODE=$(echo "$MASTER_RESULT" | tail -1) MASTER_BODY=$(echo "$MASTER_RESULT" | sed '$d') if [ "$MASTER_HTTP_CODE" = "200" ]; then log_pass "Master 同步 API 可访问 (HTTP $MASTER_HTTP_CODE)" log_info "响应: $MASTER_BODY" else log_fail "Master 同步 API 不可访问 (HTTP $MASTER_HTTP_CODE)" log_info "响应: $MASTER_BODY" echo -e " ${RED}Master 不可访问,无法继续测试${NC}" exit 1 fi # T0.2: Slave 连通性 print_section "T0.2 Slave API 连通性" SLAVE_RESULT=$(curl -s -m $TIMEOUT -w "\n%{http_code}" "$SLAVE/v1/models" \ -H "Authorization: Bearer $SLAVE_USER_KEY") SLAVE_HTTP_CODE=$(echo "$SLAVE_RESULT" | tail -1) if [ "$SLAVE_HTTP_CODE" = "200" ]; then log_pass "Slave API 可访问 (HTTP $SLAVE_HTTP_CODE)" else log_fail "Slave API 不可访问 (HTTP $SLAVE_HTTP_CODE)" echo -e " ${RED}Slave 不可访问,无法继续测试${NC}" exit 1 fi # ==================== 阶段 1: 基线余额 ==================== print_header "阶段 1: 基线余额" print_section "T1.1 查询初始余额" BALANCE_BEFORE=$(query_master_balance) if [ -n "$BALANCE_BEFORE" ]; then log_pass "Master 初始余额: $BALANCE_BEFORE" else log_fail "无法查询 Master 余额" exit 1 fi if [ "$BALANCE_BEFORE" -lt 1000 ]; then log_warn "余额较低 ($BALANCE_BEFORE),部分测试可能因余额不足而失败" fi # ==================== 阶段 2: 非流式聊天调用 ==================== print_header "阶段 2: 非流式聊天调用 (Slave → 上游 → 同步 Master)" print_section "T2.1 通过 Slave 发起非流式聊天请求" log_info "模型: $MODEL, max_tokens: 10" CHAT_RESULT=$(send_chat false 10 "Say hello in one word") CHAT_HTTP_OK=true # 检查是否有错误 if echo "$CHAT_RESULT" | grep -q '"error"'; then ERROR_MSG=$(echo "$CHAT_RESULT" | grep -o '"message":"[^"]*"' | head -1) log_fail "非流式请求失败: $ERROR_MSG" CHAT_HTTP_OK=false elif echo "$CHAT_RESULT" | grep -q '"choices"'; then log_pass "非流式请求成功" # 提取使用量 USAGE=$(echo "$CHAT_RESULT" | grep -o '"total_tokens":[0-9]*' | grep -o '[0-9]*') if [ -n "$USAGE" ]; then log_info "Token 使用量: $USAGE" fi # 提取回复内容(截断显示) CONTENT=$(echo "$CHAT_RESULT" | grep -o '"content":"[^"]*"' | head -1 | sed 's/"content":"//;s/"$//') if [ -n "$CONTENT" ]; then log_info "模型回复: ${CONTENT:0:80}" fi else log_warn "响应格式未知: ${CHAT_RESULT:0:100}" CHAT_HTTP_OK=false fi if [ "$CHAT_HTTP_OK" = true ]; then print_section "T2.2 等待同步到 Master" BALANCE_AFTER_NON_STREAM=$(wait_for_sync "$BALANCE_BEFORE" "非流式调用") print_section "T2.3 验证 Master 余额变化" if [ "$BALANCE_AFTER_NON_STREAM" != "$BALANCE_BEFORE" ]; then DELTA=$((BALANCE_BEFORE - BALANCE_AFTER_NON_STREAM)) if [ "$DELTA" -gt 0 ]; then log_pass "Master 余额已扣减: $BALANCE_BEFORE → $BALANCE_AFTER_NON_STREAM (扣除 $DELTA)" else log_fail "Master 余额变化异常: $BALANCE_BEFORE → $BALANCE_AFTER_NON_STREAM (变化 $DELTA)" fi else log_fail "Master 余额未变化,同步可能失败" fi else log_skip "T2.2-T2.3(非流式请求失败)" BALANCE_AFTER_NON_STREAM=$BALANCE_BEFORE fi # ==================== 阶段 3: 流式聊天调用 ==================== print_header "阶段 3: 流式聊天调用 (SSE)" print_section "T3.1 通过 Slave 发起流式聊天请求" log_info "模型: $MODEL, stream: true, max_tokens: 20" STREAM_RESULT=$(curl -s -m $TIMEOUT --no-buffer -X POST "$SLAVE/v1/chat/completions" \ -H "Authorization: Bearer $SLAVE_USER_KEY" \ -H "Content-Type: application/json" \ -d "{ \"model\": \"$MODEL\", \"messages\": [{\"role\": \"user\", \"content\": \"Count from 1 to 3\"}], \"max_tokens\": 20, \"stream\": true }" 2>&1) STREAM_OK=true if echo "$STREAM_RESULT" | grep -q '"error"'; then ERROR_MSG=$(echo "$STREAM_RESULT" | grep -o '"message":"[^"]*"' | head -1) log_fail "流式请求失败: $ERROR_MSG" STREAM_OK=false elif echo "$STREAM_RESULT" | grep -q 'data:'; then log_pass "流式请求成功,收到 SSE 数据" # 验证 SSE 格式 SSE_DATA_LINES=$(echo "$STREAM_RESULT" | grep -c "data:" || echo "0") HAS_DONE=$(echo "$STREAM_RESULT" | grep -c "data: \[DONE\]" || echo "0") if [ "$SSE_DATA_LINES" -gt 0 ]; then log_info "收到 $SSE_DATA_LINES 行 SSE 数据" fi if [ "$HAS_DONE" -gt 0 ]; then log_pass "流式响应包含 [DONE] 标记" else log_warn "流式响应未包含 [DONE] 标记" fi else log_warn "响应中无 SSE 数据: ${STREAM_RESULT:0:100}" STREAM_OK=false fi if [ "$STREAM_OK" = true ]; then print_section "T3.2 等待同步到 Master" BALANCE_BEFORE_STREAM=$BALANCE_AFTER_NON_STREAM BALANCE_AFTER_STREAM=$(wait_for_sync "$BALANCE_BEFORE_STREAM" "流式调用") print_section "T3.3 验证 Master 余额变化" if [ "$BALANCE_AFTER_STREAM" != "$BALANCE_BEFORE_STREAM" ]; then DELTA=$((BALANCE_BEFORE_STREAM - BALANCE_AFTER_STREAM)) if [ "$DELTA" -gt 0 ]; then log_pass "流式调用同步成功: $BALANCE_BEFORE_STREAM → $BALANCE_AFTER_STREAM (扣除 $DELTA)" else log_fail "流式调用余额变化异常: $DELTA" fi else log_warn "流式调用后 Master 余额未变化(可能 token 消耗为 0)" fi else log_skip "T3.2-T3.3(流式请求失败)" BALANCE_AFTER_STREAM=$BALANCE_AFTER_NON_STREAM fi # ==================== 阶段 4: 并发请求 ==================== print_header "阶段 4: 并发请求 ($CONCURRENT_COUNT 个)" print_section "T4.1 并发发送 $CONCURRENT_COUNT 个聊天请求" BALANCE_BEFORE_CONCURRENT=$BALANCE_AFTER_STREAM # 准备不同的 prompt 以避免缓存 PROMPTS=("What is 1+1?" "Tell me a color" "Say yes or no" "What day is it?" "Name a fruit" "What is 2+2?" "Say goodbye" "What is water?" "Name an animal" "Say OK") PIDS=() for i in $(seq 1 $CONCURRENT_COUNT); do PROMPT="${PROMPTS[$((i - 1))]}" send_chat false 10 "$PROMPT" > "/tmp/sync_test_concurrent_$i.json" 2>&1 & PIDS+=($!) done log_info "已发送 $CONCURRENT_COUNT 个并发请求,等待完成..." SUCCESS_COUNT=0 FAIL_COUNT_CONCURRENT=0 for pid in "${PIDS[@]}"; do if wait "$pid"; then ((SUCCESS_COUNT++)) || true else ((FAIL_COUNT_CONCURRENT++)) || true fi done if [ "$FAIL_COUNT_CONCURRENT" -eq 0 ]; then log_pass "全部 $CONCURRENT_COUNT 个请求完成" else log_warn "$SUCCESS_COUNT 成功, $FAIL_COUNT_CONCURRENT 失败" fi # 检查响应 print_section "T4.2 验证并发请求响应" VALID_RESPONSES=0 for i in $(seq 1 $CONCURRENT_COUNT); do if [ -f "/tmp/sync_test_concurrent_$i.json" ]; then if grep -q '"choices"' "/tmp/sync_test_concurrent_$i.json"; then ((VALID_RESPONSES++)) || true fi fi done log_info "有效响应: $VALID_RESPONSES / $CONCURRENT_COUNT" if [ "$VALID_RESPONSES" -gt 0 ]; then print_section "T4.3 等待同步到 Master" BALANCE_AFTER_CONCURRENT=$(wait_for_sync "$BALANCE_BEFORE_CONCURRENT" "并发调用") print_section "T4.4 验证 Master 余额变化" if [ "$BALANCE_AFTER_CONCURRENT" != "$BALANCE_BEFORE_CONCURRENT" ]; then DELTA=$((BALANCE_BEFORE_CONCURRENT - BALANCE_AFTER_CONCURRENT)) if [ "$DELTA" -gt 0 ]; then log_pass "并发调用同步成功: $BALANCE_BEFORE_CONCURRENT → $BALANCE_AFTER_CONCURRENT (共扣除 $DELTA)" else log_fail "并发调用余额变化异常: $DELTA" fi else log_warn "并发调用后 Master 余额未变化" fi else log_skip "T4.3-T4.4(无有效并发响应)" BALANCE_AFTER_CONCURRENT=$BALANCE_BEFORE_CONCURRENT fi # 清理临时文件 rm -f /tmp/sync_test_concurrent_*.json # ==================== 阶段 5: 错误场景 ==================== print_header "阶段 5: 错误场景测试" # T5.1: 错误模型名 print_section "T5.1 错误模型名" ERR_RESULT=$(curl -s -m $TIMEOUT -w "\n%{http_code}" -X POST "$SLAVE/v1/chat/completions" \ -H "Authorization: Bearer $SLAVE_USER_KEY" \ -H "Content-Type: application/json" \ -d '{"model":"nonexistent-model-xyz-12345","messages":[{"role":"user","content":"hi"}],"max_tokens":5}') ERR_HTTP=$(echo "$ERR_RESULT" | tail -1) ERR_BODY=$(echo "$ERR_RESULT" | sed '$d') if [ "$ERR_HTTP" != "200" ]; then log_pass "错误模型名被拒绝 (HTTP $ERR_HTTP)" else log_fail "错误模型名未被拒绝 (HTTP $ERR_HTTP)" fi log_info "响应: ${ERR_BODY:0:100}" # T5.2: 无认证 print_section "T5.2 无认证请求" NO_AUTH_RESULT=$(curl -s -m $TIMEOUT -w "\n%{http_code}" -X POST "$SLAVE/v1/chat/completions" \ -H "Content-Type: application/json" \ -d '{"model":"gpt-3.5-turbo","messages":[{"role":"user","content":"hi"}],"max_tokens":5}') NO_AUTH_HTTP=$(echo "$NO_AUTH_RESULT" | tail -1) if [ "$NO_AUTH_HTTP" != "200" ]; then log_pass "无认证请求被拒绝 (HTTP $NO_AUTH_HTTP)" else log_fail "无认证请求未被拒绝 (HTTP $NO_AUTH_HTTP)" fi # T5.3: 错误 API Key print_section "T5.3 错误 API Key" BAD_KEY_RESULT=$(curl -s -m $TIMEOUT -w "\n%{http_code}" -X POST "$SLAVE/v1/chat/completions" \ -H "Authorization: Bearer sk-invalid-key-12345" \ -H "Content-Type: application/json" \ -d '{"model":"gpt-3.5-turbo","messages":[{"role":"user","content":"hi"}],"max_tokens":5}') BAD_KEY_HTTP=$(echo "$BAD_KEY_RESULT" | tail -1) if [ "$BAD_KEY_HTTP" != "200" ]; then log_pass "错误 API Key 被拒绝 (HTTP $BAD_KEY_HTTP)" else log_fail "错误 API Key 未被拒绝 (HTTP $BAD_KEY_HTTP)" fi # T5.4: 空消息 print_section "T5.4 空消息列表" EMPTY_MSG_RESULT=$(curl -s -m $TIMEOUT -w "\n%{http_code}" -X POST "$SLAVE/v1/chat/completions" \ -H "Authorization: Bearer $SLAVE_USER_KEY" \ -H "Content-Type: application/json" \ -d '{"model":"gpt-3.5-turbo","messages":[],"max_tokens":5}') EMPTY_MSG_HTTP=$(echo "$EMPTY_MSG_RESULT" | tail -1) if [ "$EMPTY_MSG_HTTP" != "200" ]; then log_pass "空消息被拒绝 (HTTP $EMPTY_MSG_HTTP)" else log_warn "空消息未被拒绝 (HTTP $EMPTY_MSG_HTTP) — 某些实现可能允许" fi # ==================== 阶段 6: 直接同步 API 测试 ==================== print_header "阶段 6: 直接同步 API 测试" BALANCE_BEFORE_DIRECT=$(query_master_balance) log_info "直接测试前 Master 余额: $BALANCE_BEFORE_DIRECT" # T6.1: 批量扣费 print_section "T6.1 批量扣费 ($DEDUCT_AMOUNT)" REQUEST_ID="req-e2e-$(date +%s%N)" DEDUCT_RESULT=$(curl -s -m $TIMEOUT -X POST "$MASTER/api/internal/sync/quota/batch-deduct" \ -H "Content-Type: application/json" \ -H "X-Sync-API-Key: $SYNC_API_KEY" \ -H "X-Sync-Node: test-node" \ -d "{\"records\": [{\"user_id\": $USER_ID, \"request_id\": \"$REQUEST_ID\", \"quota\": $DEDUCT_AMOUNT}]}") log_info "响应: $DEDUCT_RESULT" if echo "$DEDUCT_RESULT" | grep -qi "success\|true\|200"; then log_pass "批量扣费请求成功" else log_fail "批量扣费请求失败: $DEDUCT_RESULT" fi # T6.2: 幂等性测试(相同 request_id) print_section "T6.2 幂等性测试(相同 request_id)" IDEMPOTENT_RESULT=$(curl -s -m $TIMEOUT -X POST "$MASTER/api/internal/sync/quota/batch-deduct" \ -H "Content-Type: application/json" \ -H "X-Sync-API-Key: $SYNC_API_KEY" \ -H "X-Sync-Node: test-node" \ -d "{\"records\": [{\"user_id\": $USER_ID, \"request_id\": \"$REQUEST_ID\", \"quota\": $DEDUCT_AMOUNT}]}") log_info "响应: $IDEMPOTENT_RESULT" BALANCE_AFTER_FIRST=$(query_master_balance) log_info "幂等测试后余额: $BALANCE_AFTER_FIRST" # 再查一次确认没变 sleep 1 BALANCE_AFTER_IDEMPOTENT=$(query_master_balance) if [ "$BALANCE_AFTER_IDEMPOTENT" = "$BALANCE_AFTER_FIRST" ]; then log_pass "幂等性验证通过:重复请求未重复扣费" else log_fail "幂等性验证失败:余额发生了变化 ($BALANCE_AFTER_FIRST → $BALANCE_AFTER_IDEMPOTENT)" fi # T6.3: 精确金额验证 print_section "T6.3 精确扣费金额验证" log_info "扣费前: $BALANCE_BEFORE_DIRECT" log_info "扣费后: $BALANCE_AFTER_IDEMPOTENT" ACTUAL_DEDUCT=$((BALANCE_BEFORE_DIRECT - BALANCE_AFTER_IDEMPOTENT)) if [ "$ACTUAL_DEDUCT" -eq $DEDUCT_AMOUNT ]; then log_pass "精确金额验证通过:扣除了 $ACTUAL_DEDUCT (期望 $DEDUCT_AMOUNT)" else log_fail "精确金额验证失败:扣除了 $ACTUAL_DEDUCT (期望 $DEDUCT_AMOUNT)" fi # ==================== 阶段 7: 总结报告 ==================== print_header "测试总结" FINAL_BALANCE=$(query_master_balance) echo -e " ${BOLD}测试统计:${NC}" echo -e " ${GREEN}PASS${NC}: $PASS_COUNT" echo -e " ${RED}FAIL${NC}: $FAIL_COUNT" echo -e " ${YELLOW}SKIP${NC}: $SKIP_COUNT" echo -e " 总计: $TOTAL_TESTS" echo "" echo -e " ${BOLD}余额变化:${NC}" echo -e " 初始余额: $BALANCE_BEFORE" echo -e " 最终余额: $FINAL_BALANCE" echo -e " 总扣除: $((BALANCE_BEFORE - FINAL_BALANCE))" echo "" echo -e " 测试模型: $MODEL" echo "" if [ "$FAIL_COUNT" -eq 0 ]; then echo -e " ${GREEN}${BOLD}全部测试通过!${NC}" else echo -e " ${RED}${BOLD}存在 $FAIL_COUNT 个失败${NC}" fi echo "" echo -e "${CYAN}============================================${NC}" # 返回退出码 if [ "$FAIL_COUNT" -gt 0 ]; then exit 1 fi exit 0