Record CLI parameters (prompt, stream, duration, burst, concurrency, probe_seconds, window_offset_ms) in report JSON so test conditions can be fully reproduced from the report file alone.main
| @@ -3,10 +3,10 @@ use crate::benchmarks::judge; | |||||
| use crate::config::AppConfig; | use crate::config::AppConfig; | ||||
| use crate::metrics::{LatencySummary, Metrics, MetricsSummary}; | use crate::metrics::{LatencySummary, Metrics, MetricsSummary}; | ||||
| use crate::report::{ | use crate::report::{ | ||||
| BenchmarkReport, BenchmarkSummaryReport, DatasetReport, LatencyReport, LimiterInferenceKind, | |||||
| LimiterInferenceReport, PhaseSummaryReport, ProbeSecondReport, RpmModeDetailReport, RpmReport, | |||||
| RpmRunReport, RpmSummaryReport, RunReport, WindowBoundaryReport, WrongCaseReport, | |||||
| write_benchmark_report, write_rpm_report, | |||||
| BenchmarkParamsReport, BenchmarkReport, BenchmarkSummaryReport, DatasetReport, LatencyReport, | |||||
| LimiterInferenceKind, LimiterInferenceReport, PhaseSummaryReport, ProbeSecondReport, | |||||
| RpmModeDetailReport, RpmParamsReport, RpmReport, RpmRunReport, RpmSummaryReport, RunReport, | |||||
| WindowBoundaryReport, WrongCaseReport, write_benchmark_report, write_rpm_report, | |||||
| }; | }; | ||||
| use crate::rpm_modes::{ | use crate::rpm_modes::{ | ||||
| ProbePhase, RpmMode, ScheduledProbe, burst_schedule, sliding_window_schedule, | ProbePhase, RpmMode, ScheduledProbe, burst_schedule, sliding_window_schedule, | ||||
| @@ -287,6 +287,7 @@ async fn run_aime_benchmark( | |||||
| benchmark: "aime2026", | benchmark: "aime2026", | ||||
| provider: provider_name, | provider: provider_name, | ||||
| model, | model, | ||||
| stream: base_request.stream, | |||||
| dataset, | dataset, | ||||
| started_at, | started_at, | ||||
| duration_ms: started.elapsed().as_millis(), | duration_ms: started.elapsed().as_millis(), | ||||
| @@ -372,6 +373,7 @@ async fn run_gpqa_benchmark( | |||||
| benchmark: "gpqa-diamond", | benchmark: "gpqa-diamond", | ||||
| provider: provider_name, | provider: provider_name, | ||||
| model, | model, | ||||
| stream: base_request.stream, | |||||
| dataset, | dataset, | ||||
| started_at, | started_at, | ||||
| duration_ms: started.elapsed().as_millis(), | duration_ms: started.elapsed().as_millis(), | ||||
| @@ -414,9 +416,11 @@ async fn run_rpm(config_path: PathBuf, options: RpmCommandOptions) -> Result<()> | |||||
| let model = options | let model = options | ||||
| .model | .model | ||||
| .unwrap_or_else(|| provider_config.default_model.clone()); | .unwrap_or_else(|| provider_config.default_model.clone()); | ||||
| let stream_enabled = options.stream.unwrap_or(provider_config.stream); | |||||
| let concurrency = options.concurrency.unwrap_or(mode_plan.default_concurrency); | |||||
| let request = ModelRequest { | let request = ModelRequest { | ||||
| prompt: options.prompt, | |||||
| stream: options.stream.unwrap_or(provider_config.stream), | |||||
| prompt: options.prompt.clone(), | |||||
| stream: stream_enabled, | |||||
| ..request_template(&provider_config, &model, 0.0, 1024) | ..request_template(&provider_config, &model, 0.0, 1024) | ||||
| }; | }; | ||||
| let started_at = Utc::now(); | let started_at = Utc::now(); | ||||
| @@ -428,7 +432,7 @@ async fn run_rpm(config_path: PathBuf, options: RpmCommandOptions) -> Result<()> | |||||
| provider_config.protocol, | provider_config.protocol, | ||||
| request, | request, | ||||
| mode_plan.probes, | mode_plan.probes, | ||||
| options.concurrency.unwrap_or(mode_plan.default_concurrency), | |||||
| concurrency, | |||||
| ) | ) | ||||
| .await; | .await; | ||||
| @@ -446,6 +450,15 @@ async fn run_rpm(config_path: PathBuf, options: RpmCommandOptions) -> Result<()> | |||||
| benchmark: "rpm".to_string(), | benchmark: "rpm".to_string(), | ||||
| provider: provider_name, | provider: provider_name, | ||||
| model, | model, | ||||
| params: RpmParamsReport { | |||||
| prompt: options.prompt, | |||||
| stream: stream_enabled, | |||||
| duration: options.duration, | |||||
| burst: options.burst, | |||||
| concurrency, | |||||
| probe_seconds: options.probe_seconds, | |||||
| window_offset_ms: options.window_offset_ms, | |||||
| }, | |||||
| run: RpmRunReport { | run: RpmRunReport { | ||||
| started_at, | started_at, | ||||
| duration_ms: started.elapsed().as_millis(), | duration_ms: started.elapsed().as_millis(), | ||||
| @@ -848,6 +861,7 @@ struct BenchmarkReportInput { | |||||
| benchmark: &'static str, | benchmark: &'static str, | ||||
| provider: String, | provider: String, | ||||
| model: String, | model: String, | ||||
| stream: bool, | |||||
| dataset: DatasetReport, | dataset: DatasetReport, | ||||
| started_at: chrono::DateTime<Utc>, | started_at: chrono::DateTime<Utc>, | ||||
| duration_ms: u128, | duration_ms: u128, | ||||
| @@ -862,6 +876,9 @@ fn benchmark_report(input: BenchmarkReportInput) -> BenchmarkReport { | |||||
| benchmark: input.benchmark.to_string(), | benchmark: input.benchmark.to_string(), | ||||
| provider: input.provider, | provider: input.provider, | ||||
| model: input.model, | model: input.model, | ||||
| params: BenchmarkParamsReport { | |||||
| stream: input.stream, | |||||
| }, | |||||
| dataset: input.dataset, | dataset: input.dataset, | ||||
| run: RunReport { | run: RunReport { | ||||
| started_at: input.started_at, | started_at: input.started_at, | ||||
| @@ -55,6 +55,7 @@ pub struct BenchmarkReport { | |||||
| pub benchmark: String, | pub benchmark: String, | ||||
| pub provider: String, | pub provider: String, | ||||
| pub model: String, | pub model: String, | ||||
| pub params: BenchmarkParamsReport, | |||||
| pub dataset: DatasetReport, | pub dataset: DatasetReport, | ||||
| pub run: RunReport, | pub run: RunReport, | ||||
| pub summary: BenchmarkSummaryReport, | pub summary: BenchmarkSummaryReport, | ||||
| @@ -62,6 +63,22 @@ pub struct BenchmarkReport { | |||||
| pub wrong_cases: Vec<WrongCaseReport>, | pub wrong_cases: Vec<WrongCaseReport>, | ||||
| } | } | ||||
| #[derive(Debug, Clone, Serialize)] | |||||
| pub struct BenchmarkParamsReport { | |||||
| pub stream: bool, | |||||
| } | |||||
| #[derive(Debug, Clone, Serialize)] | |||||
| pub struct RpmParamsReport { | |||||
| pub prompt: String, | |||||
| pub stream: bool, | |||||
| pub duration: String, | |||||
| pub burst: Option<u32>, | |||||
| pub concurrency: usize, | |||||
| pub probe_seconds: Option<u64>, | |||||
| pub window_offset_ms: u64, | |||||
| } | |||||
| #[derive(Debug, Clone, Serialize)] | #[derive(Debug, Clone, Serialize)] | ||||
| pub struct RpmRunReport { | pub struct RpmRunReport { | ||||
| pub started_at: DateTime<Utc>, | pub started_at: DateTime<Utc>, | ||||
| @@ -87,6 +104,7 @@ pub struct RpmReport { | |||||
| pub mode: String, | pub mode: String, | ||||
| pub provider: String, | pub provider: String, | ||||
| pub model: String, | pub model: String, | ||||
| pub params: RpmParamsReport, | |||||
| pub run: RpmRunReport, | pub run: RpmRunReport, | ||||
| pub summary: RpmSummaryReport, | pub summary: RpmSummaryReport, | ||||
| pub mode_detail: Option<RpmModeDetailReport>, | pub mode_detail: Option<RpmModeDetailReport>, | ||||
| @@ -235,6 +253,7 @@ mod tests { | |||||
| benchmark: "aime2026".to_string(), | benchmark: "aime2026".to_string(), | ||||
| provider: "openai".to_string(), | provider: "openai".to_string(), | ||||
| model: "gpt/test".to_string(), | model: "gpt/test".to_string(), | ||||
| params: BenchmarkParamsReport { stream: false }, | |||||
| dataset: DatasetReport { | dataset: DatasetReport { | ||||
| source: "local".to_string(), | source: "local".to_string(), | ||||
| split: "train".to_string(), | split: "train".to_string(), | ||||
| @@ -278,11 +297,20 @@ mod tests { | |||||
| } | } | ||||
| #[test] | #[test] | ||||
| fn rpm_report_does_not_serialize_prompt() { | |||||
| fn rpm_report_serializes_params() { | |||||
| let report = RpmReport { | let report = RpmReport { | ||||
| benchmark: "rpm".to_string(), | benchmark: "rpm".to_string(), | ||||
| provider: "openai".to_string(), | provider: "openai".to_string(), | ||||
| model: "gpt/test".to_string(), | model: "gpt/test".to_string(), | ||||
| params: RpmParamsReport { | |||||
| prompt: "Hi".to_string(), | |||||
| stream: false, | |||||
| duration: "60s".to_string(), | |||||
| burst: None, | |||||
| concurrency: 10, | |||||
| probe_seconds: None, | |||||
| window_offset_ms: 0, | |||||
| }, | |||||
| run: RpmRunReport { | run: RpmRunReport { | ||||
| started_at: Utc.with_ymd_and_hms(2026, 5, 6, 1, 2, 3).unwrap(), | started_at: Utc.with_ymd_and_hms(2026, 5, 6, 1, 2, 3).unwrap(), | ||||
| duration_ms: 1000, | duration_ms: 1000, | ||||
| @@ -313,8 +341,9 @@ mod tests { | |||||
| let json = serde_json::to_string(&report).expect("serialize report"); | let json = serde_json::to_string(&report).expect("serialize report"); | ||||
| assert!(!json.contains("sensitive prompt")); | |||||
| assert!(!json.contains("\"prompt\"")); | |||||
| assert!(json.contains("\"prompt\":\"Hi\"")); | |||||
| assert!(json.contains("\"stream\":false")); | |||||
| assert!(json.contains("\"duration\":\"60s\"")); | |||||
| } | } | ||||
| #[test] | #[test] | ||||
| @@ -324,6 +353,15 @@ mod tests { | |||||
| mode: "token-bucket".to_string(), | mode: "token-bucket".to_string(), | ||||
| provider: "anthropic".to_string(), | provider: "anthropic".to_string(), | ||||
| model: "claude/test".to_string(), | model: "claude/test".to_string(), | ||||
| params: RpmParamsReport { | |||||
| prompt: "Hi".to_string(), | |||||
| stream: true, | |||||
| duration: "90s".to_string(), | |||||
| burst: Some(50), | |||||
| concurrency: 10, | |||||
| probe_seconds: Some(90), | |||||
| window_offset_ms: 200, | |||||
| }, | |||||
| run: RpmRunReport { | run: RpmRunReport { | ||||
| started_at: Utc.with_ymd_and_hms(2026, 5, 6, 1, 2, 3).unwrap(), | started_at: Utc.with_ymd_and_hms(2026, 5, 6, 1, 2, 3).unwrap(), | ||||
| duration_ms: 2000, | duration_ms: 2000, | ||||