Просмотр исходного кода

feat: add params field to RPM and benchmark reports

Record CLI parameters (prompt, stream, duration, burst, concurrency,
probe_seconds, window_offset_ms) in report JSON so test conditions
can be fully reproduced from the report file alone.
main
orangels 1 неделю назад
Родитель
Сommit
527b1e8fc7
2 измененных файлов: 65 добавлений и 10 удалений
  1. +24
    -7
      src/cli.rs
  2. +41
    -3
      src/report.rs

+ 24
- 7
src/cli.rs Просмотреть файл

@@ -3,10 +3,10 @@ use crate::benchmarks::judge;
use crate::config::AppConfig;
use crate::metrics::{LatencySummary, Metrics, MetricsSummary};
use crate::report::{
BenchmarkReport, BenchmarkSummaryReport, DatasetReport, LatencyReport, LimiterInferenceKind,
LimiterInferenceReport, PhaseSummaryReport, ProbeSecondReport, RpmModeDetailReport, RpmReport,
RpmRunReport, RpmSummaryReport, RunReport, WindowBoundaryReport, WrongCaseReport,
write_benchmark_report, write_rpm_report,
BenchmarkParamsReport, BenchmarkReport, BenchmarkSummaryReport, DatasetReport, LatencyReport,
LimiterInferenceKind, LimiterInferenceReport, PhaseSummaryReport, ProbeSecondReport,
RpmModeDetailReport, RpmParamsReport, RpmReport, RpmRunReport, RpmSummaryReport, RunReport,
WindowBoundaryReport, WrongCaseReport, write_benchmark_report, write_rpm_report,
};
use crate::rpm_modes::{
ProbePhase, RpmMode, ScheduledProbe, burst_schedule, sliding_window_schedule,
@@ -287,6 +287,7 @@ async fn run_aime_benchmark(
benchmark: "aime2026",
provider: provider_name,
model,
stream: base_request.stream,
dataset,
started_at,
duration_ms: started.elapsed().as_millis(),
@@ -372,6 +373,7 @@ async fn run_gpqa_benchmark(
benchmark: "gpqa-diamond",
provider: provider_name,
model,
stream: base_request.stream,
dataset,
started_at,
duration_ms: started.elapsed().as_millis(),
@@ -414,9 +416,11 @@ async fn run_rpm(config_path: PathBuf, options: RpmCommandOptions) -> Result<()>
let model = options
.model
.unwrap_or_else(|| provider_config.default_model.clone());
let stream_enabled = options.stream.unwrap_or(provider_config.stream);
let concurrency = options.concurrency.unwrap_or(mode_plan.default_concurrency);
let request = ModelRequest {
prompt: options.prompt,
stream: options.stream.unwrap_or(provider_config.stream),
prompt: options.prompt.clone(),
stream: stream_enabled,
..request_template(&provider_config, &model, 0.0, 1024)
};
let started_at = Utc::now();
@@ -428,7 +432,7 @@ async fn run_rpm(config_path: PathBuf, options: RpmCommandOptions) -> Result<()>
provider_config.protocol,
request,
mode_plan.probes,
options.concurrency.unwrap_or(mode_plan.default_concurrency),
concurrency,
)
.await;

@@ -446,6 +450,15 @@ async fn run_rpm(config_path: PathBuf, options: RpmCommandOptions) -> Result<()>
benchmark: "rpm".to_string(),
provider: provider_name,
model,
params: RpmParamsReport {
prompt: options.prompt,
stream: stream_enabled,
duration: options.duration,
burst: options.burst,
concurrency,
probe_seconds: options.probe_seconds,
window_offset_ms: options.window_offset_ms,
},
run: RpmRunReport {
started_at,
duration_ms: started.elapsed().as_millis(),
@@ -848,6 +861,7 @@ struct BenchmarkReportInput {
benchmark: &'static str,
provider: String,
model: String,
stream: bool,
dataset: DatasetReport,
started_at: chrono::DateTime<Utc>,
duration_ms: u128,
@@ -862,6 +876,9 @@ fn benchmark_report(input: BenchmarkReportInput) -> BenchmarkReport {
benchmark: input.benchmark.to_string(),
provider: input.provider,
model: input.model,
params: BenchmarkParamsReport {
stream: input.stream,
},
dataset: input.dataset,
run: RunReport {
started_at: input.started_at,


+ 41
- 3
src/report.rs Просмотреть файл

@@ -55,6 +55,7 @@ pub struct BenchmarkReport {
pub benchmark: String,
pub provider: String,
pub model: String,
pub params: BenchmarkParamsReport,
pub dataset: DatasetReport,
pub run: RunReport,
pub summary: BenchmarkSummaryReport,
@@ -62,6 +63,22 @@ pub struct BenchmarkReport {
pub wrong_cases: Vec<WrongCaseReport>,
}

#[derive(Debug, Clone, Serialize)]
pub struct BenchmarkParamsReport {
pub stream: bool,
}

#[derive(Debug, Clone, Serialize)]
pub struct RpmParamsReport {
pub prompt: String,
pub stream: bool,
pub duration: String,
pub burst: Option<u32>,
pub concurrency: usize,
pub probe_seconds: Option<u64>,
pub window_offset_ms: u64,
}

#[derive(Debug, Clone, Serialize)]
pub struct RpmRunReport {
pub started_at: DateTime<Utc>,
@@ -87,6 +104,7 @@ pub struct RpmReport {
pub mode: String,
pub provider: String,
pub model: String,
pub params: RpmParamsReport,
pub run: RpmRunReport,
pub summary: RpmSummaryReport,
pub mode_detail: Option<RpmModeDetailReport>,
@@ -235,6 +253,7 @@ mod tests {
benchmark: "aime2026".to_string(),
provider: "openai".to_string(),
model: "gpt/test".to_string(),
params: BenchmarkParamsReport { stream: false },
dataset: DatasetReport {
source: "local".to_string(),
split: "train".to_string(),
@@ -278,11 +297,20 @@ mod tests {
}

#[test]
fn rpm_report_does_not_serialize_prompt() {
fn rpm_report_serializes_params() {
let report = RpmReport {
benchmark: "rpm".to_string(),
provider: "openai".to_string(),
model: "gpt/test".to_string(),
params: RpmParamsReport {
prompt: "Hi".to_string(),
stream: false,
duration: "60s".to_string(),
burst: None,
concurrency: 10,
probe_seconds: None,
window_offset_ms: 0,
},
run: RpmRunReport {
started_at: Utc.with_ymd_and_hms(2026, 5, 6, 1, 2, 3).unwrap(),
duration_ms: 1000,
@@ -313,8 +341,9 @@ mod tests {

let json = serde_json::to_string(&report).expect("serialize report");

assert!(!json.contains("sensitive prompt"));
assert!(!json.contains("\"prompt\""));
assert!(json.contains("\"prompt\":\"Hi\""));
assert!(json.contains("\"stream\":false"));
assert!(json.contains("\"duration\":\"60s\""));
}

#[test]
@@ -324,6 +353,15 @@ mod tests {
mode: "token-bucket".to_string(),
provider: "anthropic".to_string(),
model: "claude/test".to_string(),
params: RpmParamsReport {
prompt: "Hi".to_string(),
stream: true,
duration: "90s".to_string(),
burst: Some(50),
concurrency: 10,
probe_seconds: Some(90),
window_offset_ms: 200,
},
run: RpmRunReport {
started_at: Utc.with_ymd_and_hms(2026, 5, 6, 1, 2, 3).unwrap(),
duration_ms: 2000,


Загрузка…
Отмена
Сохранить