diff --git a/.gitignore b/.gitignore index c10ea52..729ca69 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target /data/benchmarks /reports +/outputs diff --git a/README.md b/README.md index 331186d..3fae4c3 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,8 @@ Benchmark and RPM commands print a terminal summary with success counts, failure Benchmark reports include `wrong_cases`, with each wrong case containing the case id, question, expected answer, extracted actual answer, and raw model output. RPM reports include request counts, mode, target RPM, observed RPM, latency, error counts, and mode-specific details such as burst summaries, probe summaries, window-boundary summaries, and optional limiter inference. +Use `--debug-raw` with `check`, `bench`, or `rpm` to write upstream raw responses under `outputs/debug/`. Non-streaming requests save the raw JSON body, and streaming requests save the raw SSE lines. The directory is ignored by git and can help diagnose relay-side response rewriting. + ## Comparing Scores Use these results as relay benchmark signals, not absolute proof by themselves. To compare against official scores or another run, align the same dataset and source, prompt text, temperature, `max_tokens`, sample limit, and scoring logic. Differences in any of those inputs can make the reported accuracy diverge from official numbers or other benchmark harnesses. diff --git a/docs/USAGE.zh-CN.md b/docs/USAGE.zh-CN.md index 09ca06b..e025b5a 100644 --- a/docs/USAGE.zh-CN.md +++ b/docs/USAGE.zh-CN.md @@ -356,6 +356,24 @@ ls -lt reports | head jq . reports/<报告文件>.json ``` +### 原始响应 Debug + +如果需要排查中转站是否改写了模型响应,可以开启 `--debug-raw`: + +```bash +cargo run -- check --provider anthropic --stream --debug-raw --prompt "hello" +cargo run -- bench aime2026 --provider anthropic --stream --debug-raw --limit 3 +cargo run -- rpm --provider anthropic --rpm 60 --duration 30s --stream --debug-raw --prompt "hello" +``` + +开启后,原始响应会写到: + +```text +outputs/debug/ +``` + +非流式请求保存完整 JSON body;流式请求保存原始 SSE 行,包括 `event:` 和 `data:`。文件不包含 API token,但可能包含模型输出内容,所以 `outputs/` 不会提交到 git。 + benchmark report 包含: - benchmark diff --git a/docs/testing-guide.md b/docs/testing-guide.md index 6c23afa..60a846f 100644 --- a/docs/testing-guide.md +++ b/docs/testing-guide.md @@ -195,6 +195,25 @@ cargo run -- bench aime2026 --provider anthropic --stream 如果 provider config 中已设置 `stream: true`,则默认使用流式,无需额外传参。CLI `--stream` 参数优先级高于 config。 +## 原始响应 Debug 模式 + +排查中转站是否改写响应时,可以加 `--debug-raw`: + +```bash +cargo run -- check --provider anthropic --stream --debug-raw --prompt "Reply with pong." +cargo run -- bench aime2026 --provider anthropic --stream --debug-raw --limit 3 +cargo run -- rpm --provider anthropic --rpm 60 --duration 30s --stream --debug-raw --prompt "Hi" +``` + +开启后,程序会把上游原始响应写到 `outputs/debug/`: + +- 非流式请求保存完整 JSON body +- 流式请求保存原始 SSE 行,包括 `event:` 和 `data:` +- 文件不包含 API token +- 文件可能包含模型输出内容,`outputs/` 不会提交到 git + +如果 `report.json` 的 `raw_output` 中出现 ``,可以查看 debug 文件判断它来自 `text_delta`,还是上游返回了其他 thinking 事件。 + --- ## 故障排查 diff --git a/src/cli.rs b/src/cli.rs index 3a1cd53..afc0d00 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -13,7 +13,7 @@ use crate::rpm_modes::{ ProbePhase, RpmMode, ScheduledProbe, burst_schedule, sliding_window_schedule, sustained_schedule, token_bucket_schedule, window_boundary_plan, }; -use crate::runner::{ModelRequest, run_model_request}; +use crate::runner::{ModelRequest, RawDebugConfig, run_model_request}; use anyhow::{Context, Result, bail}; use chrono::Utc; use clap::{Parser, Subcommand}; @@ -49,6 +49,8 @@ pub enum Command { prompt: String, #[arg(long, num_args = 0..=1, default_missing_value = "true")] stream: Option, + #[arg(long)] + debug_raw: bool, }, Dataset { #[command(subcommand)] @@ -83,6 +85,8 @@ pub enum Command { prompt: String, #[arg(long, num_args = 0..=1, default_missing_value = "true")] stream: Option, + #[arg(long)] + debug_raw: bool, }, } @@ -108,6 +112,8 @@ pub enum BenchCommand { stream: Option, #[arg(long, default_value_t = 32768)] max_tokens: u32, + #[arg(long)] + debug_raw: bool, }, GpqaDiamond { #[arg(long, default_value = "config.yaml")] @@ -124,6 +130,8 @@ pub enum BenchCommand { stream: Option, #[arg(long, default_value_t = 32768)] max_tokens: u32, + #[arg(long)] + debug_raw: bool, }, } @@ -135,17 +143,21 @@ pub async fn dispatch(cli: Cli) -> Result<()> { model, prompt, stream, + debug_raw, } => { let config = AppConfig::load(&config)?; - let provider = config.resolved_provider(provider.as_deref())?; + let provider_name = provider_name(&config, provider.as_deref())?; + let provider = config.resolved_provider(Some(&provider_name))?; + let model = model.unwrap_or_else(|| provider.default_model.clone()); let request = ModelRequest { base_url: provider.base_url.clone(), api_token: provider.api_token.clone(), - model: model.unwrap_or_else(|| provider.default_model.clone()), + model: model.clone(), prompt, temperature: 0.0, max_tokens: 1024, stream: stream.unwrap_or(provider.stream), + raw_debug: raw_debug_config(debug_raw, &provider_name, &model), }; let response = run_model_request(provider.protocol, request).await?; @@ -180,6 +192,7 @@ pub async fn dispatch(cli: Cli) -> Result<()> { concurrency, prompt, stream, + debug_raw, } => { run_rpm( config, @@ -195,6 +208,7 @@ pub async fn dispatch(cli: Cli) -> Result<()> { concurrency, prompt, stream, + debug_raw, }, ) .await @@ -212,6 +226,7 @@ async fn dispatch_bench(command: BenchCommand) -> Result<()> { limit, stream, max_tokens, + debug_raw, } => { run_aime_benchmark( config, @@ -221,6 +236,7 @@ async fn dispatch_bench(command: BenchCommand) -> Result<()> { limit, stream, max_tokens, + debug_raw, ) .await } @@ -232,6 +248,7 @@ async fn dispatch_bench(command: BenchCommand) -> Result<()> { limit, stream, max_tokens, + debug_raw, } => { run_gpqa_benchmark( config, @@ -241,6 +258,7 @@ async fn dispatch_bench(command: BenchCommand) -> Result<()> { limit, stream, max_tokens, + debug_raw, ) .await } @@ -255,6 +273,7 @@ async fn run_aime_benchmark( limit: Option, stream: Option, max_tokens: u32, + debug_raw: bool, ) -> Result<()> { let config = AppConfig::load(&config_path)?; let provider_name = provider_name(&config, provider.as_deref())?; @@ -275,6 +294,7 @@ async fn run_aime_benchmark( let started = Instant::now(); let mut base_request = request_template(&provider_config, &model, 0.0, max_tokens); base_request.stream = stream.unwrap_or(provider_config.stream); + base_request.raw_debug = raw_debug_config(debug_raw, &provider_name, &model); let protocol = provider_config.protocol; let pb = ProgressBar::new(total); @@ -364,6 +384,7 @@ async fn run_gpqa_benchmark( limit: Option, stream: Option, max_tokens: u32, + debug_raw: bool, ) -> Result<()> { let config = AppConfig::load(&config_path)?; let provider_name = provider_name(&config, provider.as_deref())?; @@ -384,6 +405,7 @@ async fn run_gpqa_benchmark( let started = Instant::now(); let mut base_request = request_template(&provider_config, &model, 0.0, max_tokens); base_request.stream = stream.unwrap_or(provider_config.stream); + base_request.raw_debug = raw_debug_config(debug_raw, &provider_name, &model); let protocol = provider_config.protocol; let pb = ProgressBar::new(total); @@ -479,6 +501,7 @@ struct RpmCommandOptions { concurrency: Option, prompt: String, stream: Option, + debug_raw: bool, } async fn run_rpm(config_path: PathBuf, options: RpmCommandOptions) -> Result<()> { @@ -501,6 +524,7 @@ async fn run_rpm(config_path: PathBuf, options: RpmCommandOptions) -> Result<()> let request = ModelRequest { prompt: options.prompt.clone(), stream: stream_enabled, + raw_debug: raw_debug_config(options.debug_raw, &provider_name, &model), ..request_template(&provider_config, &model, 0.0, 1024) }; let started_at = Utc::now(); @@ -921,6 +945,15 @@ fn provider_name(config: &AppConfig, provider: Option<&str>) -> Result { } } +fn raw_debug_config(enabled: bool, provider: &str, model: &str) -> Option { + enabled.then(|| { + RawDebugConfig::new( + PathBuf::from("outputs/debug"), + format!("{provider}-{model}"), + ) + }) +} + fn request_template( provider: &crate::config::ProviderConfig, model: &str, @@ -935,6 +968,7 @@ fn request_template( temperature, max_tokens, stream: provider.stream, + raw_debug: None, } } @@ -1297,6 +1331,26 @@ mod tests { assert_eq!(stream, Some(false)); } + #[test] + fn check_command_parses_debug_raw() { + let cli = Cli::try_parse_from([ + "lq_token_test", + "check", + "--provider", + "anthropic", + "--prompt", + "hello", + "--debug-raw", + ]) + .expect("parse check debug raw"); + + let Command::Check { debug_raw, .. } = cli.command else { + panic!("expected check command"); + }; + + assert!(debug_raw); + } + #[test] fn rpm_command_parses_window_boundary_offset() { let cli = Cli::try_parse_from([ diff --git a/src/protocols/anthropic.rs b/src/protocols/anthropic.rs index f210dfa..35888de 100644 --- a/src/protocols/anthropic.rs +++ b/src/protocols/anthropic.rs @@ -28,6 +28,12 @@ pub async fn send(client: &Client, request: &ModelRequest) -> Result Result Result = None; let mut current_event = String::new(); let mut done = false; @@ -97,6 +110,8 @@ pub async fn send_stream(client: &Client, request: &ModelRequest) -> Result Result, _>>() + .expect("debug entries"); + assert_eq!(debug_files.len(), 1); + let raw = std::fs::read_to_string(debug_files[0].path()).expect("read raw debug file"); + assert!(raw.contains("event: content_block_delta")); + assert!(raw.contains("\"text\":\"hi \"")); } } diff --git a/src/protocols/openai.rs b/src/protocols/openai.rs index 1508878..d3cc5a2 100644 --- a/src/protocols/openai.rs +++ b/src/protocols/openai.rs @@ -27,6 +27,12 @@ pub async fn send(client: &Client, request: &ModelRequest) -> Result Result Result = None; let mut done = false; while let Some(chunk) = stream.next().await { let chunk = chunk.context("OpenAI stream interrupted")?; for line in buffer.feed(&chunk) { + raw_stream.push_str(&line); + raw_stream.push('\n'); let Some(data) = line.strip_prefix("data: ") else { continue; }; @@ -117,6 +132,13 @@ pub async fn send_stream(client: &Client, request: &ModelRequest) -> Result, _>>() + .expect("debug entries"); + assert_eq!(debug_files.len(), 1); + let raw = std::fs::read_to_string(debug_files[0].path()).expect("read raw debug file"); + assert!(raw.contains("data: {\"choices\"")); + assert!(raw.contains("data: [DONE]")); } } diff --git a/src/runner.rs b/src/runner.rs index e9bf75b..26bed5f 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -1,8 +1,12 @@ use crate::config::ProtocolKind; use crate::protocols; -use anyhow::Result; +use anyhow::{Context, Result}; +use chrono::Utc; use reqwest::Client; use std::fmt; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Instant; #[derive(Clone)] @@ -14,6 +18,48 @@ pub struct ModelRequest { pub temperature: f32, pub max_tokens: u32, pub stream: bool, + pub raw_debug: Option, +} + +#[derive(Clone)] +pub struct RawDebugConfig { + output_dir: PathBuf, + prefix: String, + counter: Arc, +} + +impl RawDebugConfig { + pub fn new(output_dir: PathBuf, prefix: String) -> Self { + Self { + output_dir, + prefix: sanitize_filename_component(&prefix), + counter: Arc::new(AtomicU64::new(0)), + } + } + + pub async fn write_response(&self, response_kind: &str, contents: &str) -> Result { + tokio::fs::create_dir_all(&self.output_dir) + .await + .with_context(|| { + format!( + "failed to create raw debug dir {}", + self.output_dir.display() + ) + })?; + let sequence = self.counter.fetch_add(1, Ordering::Relaxed) + 1; + let timestamp = Utc::now().format("%Y%m%dT%H%M%S%.3fZ"); + let filename = format!( + "{}-{}-{sequence:06}-{}.txt", + self.prefix, + timestamp, + sanitize_filename_component(response_kind) + ); + let path = self.output_dir.join(filename); + tokio::fs::write(&path, contents) + .await + .with_context(|| format!("failed to write raw debug response {}", path.display()))?; + Ok(path) + } } impl fmt::Debug for ModelRequest { @@ -27,10 +73,34 @@ impl fmt::Debug for ModelRequest { .field("temperature", &self.temperature) .field("max_tokens", &self.max_tokens) .field("stream", &self.stream) + .field("raw_debug", &self.raw_debug.is_some()) .finish() } } +fn sanitize_filename_component(value: &str) -> String { + let sanitized = value + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') { + character + } else { + '-' + } + }) + .collect::(); + let collapsed = sanitized + .split('-') + .filter(|part| !part.is_empty()) + .collect::>() + .join("-"); + if collapsed.is_empty() { + "unknown".to_string() + } else { + collapsed + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct ModelResponse { pub text: String, @@ -82,6 +152,7 @@ mod tests { temperature: 0.0, max_tokens: 1024, stream: false, + raw_debug: None, }; let debug = format!("{request:?}");