diff --git a/src/cli.rs b/src/cli.rs index 9541069..40992f6 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -3,10 +3,10 @@ use crate::benchmarks::judge; use crate::config::AppConfig; use crate::metrics::{LatencySummary, Metrics, MetricsSummary}; use crate::report::{ - BenchmarkReport, BenchmarkSummaryReport, DatasetReport, LatencyReport, LimiterInferenceKind, - LimiterInferenceReport, PhaseSummaryReport, ProbeSecondReport, RpmModeDetailReport, RpmReport, - RpmRunReport, RpmSummaryReport, RunReport, WindowBoundaryReport, WrongCaseReport, - write_benchmark_report, write_rpm_report, + BenchmarkParamsReport, BenchmarkReport, BenchmarkSummaryReport, DatasetReport, LatencyReport, + LimiterInferenceKind, LimiterInferenceReport, PhaseSummaryReport, ProbeSecondReport, + RpmModeDetailReport, RpmParamsReport, RpmReport, RpmRunReport, RpmSummaryReport, RunReport, + WindowBoundaryReport, WrongCaseReport, write_benchmark_report, write_rpm_report, }; use crate::rpm_modes::{ ProbePhase, RpmMode, ScheduledProbe, burst_schedule, sliding_window_schedule, @@ -287,6 +287,7 @@ async fn run_aime_benchmark( benchmark: "aime2026", provider: provider_name, model, + stream: base_request.stream, dataset, started_at, duration_ms: started.elapsed().as_millis(), @@ -372,6 +373,7 @@ async fn run_gpqa_benchmark( benchmark: "gpqa-diamond", provider: provider_name, model, + stream: base_request.stream, dataset, started_at, duration_ms: started.elapsed().as_millis(), @@ -414,9 +416,11 @@ async fn run_rpm(config_path: PathBuf, options: RpmCommandOptions) -> Result<()> let model = options .model .unwrap_or_else(|| provider_config.default_model.clone()); + let stream_enabled = options.stream.unwrap_or(provider_config.stream); + let concurrency = options.concurrency.unwrap_or(mode_plan.default_concurrency); let request = ModelRequest { - prompt: options.prompt, - stream: options.stream.unwrap_or(provider_config.stream), + prompt: options.prompt.clone(), + stream: stream_enabled, ..request_template(&provider_config, &model, 0.0, 1024) }; let started_at = Utc::now(); @@ -428,7 +432,7 @@ async fn run_rpm(config_path: PathBuf, options: RpmCommandOptions) -> Result<()> provider_config.protocol, request, mode_plan.probes, - options.concurrency.unwrap_or(mode_plan.default_concurrency), + concurrency, ) .await; @@ -446,6 +450,15 @@ async fn run_rpm(config_path: PathBuf, options: RpmCommandOptions) -> Result<()> benchmark: "rpm".to_string(), provider: provider_name, model, + params: RpmParamsReport { + prompt: options.prompt, + stream: stream_enabled, + duration: options.duration, + burst: options.burst, + concurrency, + probe_seconds: options.probe_seconds, + window_offset_ms: options.window_offset_ms, + }, run: RpmRunReport { started_at, duration_ms: started.elapsed().as_millis(), @@ -848,6 +861,7 @@ struct BenchmarkReportInput { benchmark: &'static str, provider: String, model: String, + stream: bool, dataset: DatasetReport, started_at: chrono::DateTime, duration_ms: u128, @@ -862,6 +876,9 @@ fn benchmark_report(input: BenchmarkReportInput) -> BenchmarkReport { benchmark: input.benchmark.to_string(), provider: input.provider, model: input.model, + params: BenchmarkParamsReport { + stream: input.stream, + }, dataset: input.dataset, run: RunReport { started_at: input.started_at, diff --git a/src/report.rs b/src/report.rs index c430a4b..97d235a 100644 --- a/src/report.rs +++ b/src/report.rs @@ -55,6 +55,7 @@ pub struct BenchmarkReport { pub benchmark: String, pub provider: String, pub model: String, + pub params: BenchmarkParamsReport, pub dataset: DatasetReport, pub run: RunReport, pub summary: BenchmarkSummaryReport, @@ -62,6 +63,22 @@ pub struct BenchmarkReport { pub wrong_cases: Vec, } +#[derive(Debug, Clone, Serialize)] +pub struct BenchmarkParamsReport { + pub stream: bool, +} + +#[derive(Debug, Clone, Serialize)] +pub struct RpmParamsReport { + pub prompt: String, + pub stream: bool, + pub duration: String, + pub burst: Option, + pub concurrency: usize, + pub probe_seconds: Option, + pub window_offset_ms: u64, +} + #[derive(Debug, Clone, Serialize)] pub struct RpmRunReport { pub started_at: DateTime, @@ -87,6 +104,7 @@ pub struct RpmReport { pub mode: String, pub provider: String, pub model: String, + pub params: RpmParamsReport, pub run: RpmRunReport, pub summary: RpmSummaryReport, pub mode_detail: Option, @@ -235,6 +253,7 @@ mod tests { benchmark: "aime2026".to_string(), provider: "openai".to_string(), model: "gpt/test".to_string(), + params: BenchmarkParamsReport { stream: false }, dataset: DatasetReport { source: "local".to_string(), split: "train".to_string(), @@ -278,11 +297,20 @@ mod tests { } #[test] - fn rpm_report_does_not_serialize_prompt() { + fn rpm_report_serializes_params() { let report = RpmReport { benchmark: "rpm".to_string(), provider: "openai".to_string(), model: "gpt/test".to_string(), + params: RpmParamsReport { + prompt: "Hi".to_string(), + stream: false, + duration: "60s".to_string(), + burst: None, + concurrency: 10, + probe_seconds: None, + window_offset_ms: 0, + }, run: RpmRunReport { started_at: Utc.with_ymd_and_hms(2026, 5, 6, 1, 2, 3).unwrap(), duration_ms: 1000, @@ -313,8 +341,9 @@ mod tests { let json = serde_json::to_string(&report).expect("serialize report"); - assert!(!json.contains("sensitive prompt")); - assert!(!json.contains("\"prompt\"")); + assert!(json.contains("\"prompt\":\"Hi\"")); + assert!(json.contains("\"stream\":false")); + assert!(json.contains("\"duration\":\"60s\"")); } #[test] @@ -324,6 +353,15 @@ mod tests { mode: "token-bucket".to_string(), provider: "anthropic".to_string(), model: "claude/test".to_string(), + params: RpmParamsReport { + prompt: "Hi".to_string(), + stream: true, + duration: "90s".to_string(), + burst: Some(50), + concurrency: 10, + probe_seconds: Some(90), + window_offset_ms: 200, + }, run: RpmRunReport { started_at: Utc.with_ymd_and_hms(2026, 5, 6, 1, 2, 3).unwrap(), duration_ms: 2000,