api模型检测
Du kan inte välja fler än 25 ämnen Ämnen måste starta med en bokstav eller siffra, kan innehålla bindestreck ('-') och vara max 35 tecken långa.

2120 rader
68 KiB

  1. use crate::benchmarks;
  2. use crate::benchmarks::judge;
  3. use crate::config::{AppConfig, ProtocolKind, ProviderThinkingConfig};
  4. use crate::metrics::{LatencySummary, Metrics, MetricsSummary};
  5. use crate::report::{
  6. BenchmarkParamsReport, BenchmarkReport, BenchmarkSummaryReport, CorrectCaseReport,
  7. DatasetReport, LatencyReport, LimiterInferenceKind, LimiterInferenceReport, PhaseSummaryReport,
  8. ProbeSecondReport, RequestDebugReport, RpmModeDetailReport, RpmParamsReport, RpmReport,
  9. RpmRunReport, RpmSummaryReport, RunReport, SentRequestParamsReport, ThinkingParamsReport,
  10. WindowBoundaryReport, WrongCaseReport, write_benchmark_report, write_rpm_report,
  11. };
  12. use crate::rpm_modes::{
  13. ProbePhase, RpmMode, ScheduledProbe, burst_schedule, sliding_window_schedule,
  14. sustained_schedule, token_bucket_schedule, window_boundary_plan,
  15. };
  16. use crate::runner::{
  17. ModelRequest, ModelResponse, RawDebugConfig, ThinkingConfig, run_model_request,
  18. };
  19. use anyhow::{Context, Result, bail};
  20. use chrono::Utc;
  21. use clap::{Parser, Subcommand};
  22. use futures::{StreamExt, stream};
  23. use indicatif::{ProgressBar, ProgressStyle};
  24. use regex::Regex;
  25. use serde_json::{Value, json};
  26. use std::collections::BTreeMap;
  27. use std::path::{Path, PathBuf};
  28. use std::time::{Duration, Instant};
  29. use tokio::time::{Instant as TokioInstant, sleep_until};
  30. #[derive(Debug, Parser)]
  31. #[command(
  32. name = "lq_token_test",
  33. version,
  34. about = "Test LLM relay protocols, RPM, and benchmark accuracy"
  35. )]
  36. pub struct Cli {
  37. #[command(subcommand)]
  38. pub command: Command,
  39. }
  40. #[derive(Debug, Subcommand)]
  41. pub enum Command {
  42. Check {
  43. #[arg(long, default_value = "config.yaml")]
  44. config: PathBuf,
  45. #[arg(long)]
  46. provider: Option<String>,
  47. #[arg(long)]
  48. model: Option<String>,
  49. #[arg(long)]
  50. prompt: String,
  51. #[arg(long, num_args = 0..=1, default_missing_value = "true")]
  52. stream: Option<bool>,
  53. #[arg(long)]
  54. debug_raw: bool,
  55. #[arg(long, num_args = 0..=1, default_missing_value = "true")]
  56. thinking: Option<bool>,
  57. #[arg(long)]
  58. thinking_type: Option<String>,
  59. #[arg(long)]
  60. thinking_budget_tokens: Option<u32>,
  61. #[arg(long)]
  62. thinking_effort: Option<String>,
  63. #[arg(long)]
  64. thinking_display: Option<String>,
  65. #[arg(long)]
  66. reasoning_effort: Option<String>,
  67. #[arg(long)]
  68. reasoning_summary: Option<String>,
  69. },
  70. Dataset {
  71. #[command(subcommand)]
  72. command: DatasetCommand,
  73. },
  74. Bench {
  75. #[command(subcommand)]
  76. command: BenchCommand,
  77. },
  78. Rpm {
  79. #[arg(long, default_value = "config.yaml")]
  80. config: PathBuf,
  81. #[arg(long, value_enum, default_value_t = RpmMode::Sustained)]
  82. mode: RpmMode,
  83. #[arg(long)]
  84. provider: Option<String>,
  85. #[arg(long)]
  86. model: Option<String>,
  87. #[arg(long)]
  88. rpm: Option<u32>,
  89. #[arg(long, default_value = "60s")]
  90. duration: String,
  91. #[arg(long)]
  92. burst: Option<u32>,
  93. #[arg(long)]
  94. probe_seconds: Option<u64>,
  95. #[arg(long, default_value_t = 500)]
  96. window_offset_ms: u64,
  97. #[arg(long)]
  98. concurrency: Option<usize>,
  99. #[arg(long)]
  100. prompt: String,
  101. #[arg(long, num_args = 0..=1, default_missing_value = "true")]
  102. stream: Option<bool>,
  103. #[arg(long)]
  104. debug_raw: bool,
  105. #[arg(long, num_args = 0..=1, default_missing_value = "true")]
  106. thinking: Option<bool>,
  107. #[arg(long)]
  108. thinking_type: Option<String>,
  109. #[arg(long)]
  110. thinking_budget_tokens: Option<u32>,
  111. #[arg(long)]
  112. thinking_effort: Option<String>,
  113. #[arg(long)]
  114. thinking_display: Option<String>,
  115. #[arg(long)]
  116. reasoning_effort: Option<String>,
  117. #[arg(long)]
  118. reasoning_summary: Option<String>,
  119. },
  120. }
  121. #[derive(Debug, Subcommand)]
  122. pub enum DatasetCommand {
  123. Fetch { dataset: String },
  124. }
  125. #[derive(Debug, Subcommand)]
  126. pub enum BenchCommand {
  127. Aime2026 {
  128. #[arg(long, default_value = "config.yaml")]
  129. config: PathBuf,
  130. #[arg(long)]
  131. provider: Option<String>,
  132. #[arg(long)]
  133. model: Option<String>,
  134. #[arg(long, default_value_t = 4)]
  135. concurrency: usize,
  136. #[arg(long)]
  137. limit: Option<usize>,
  138. #[arg(long, num_args = 0..=1, default_missing_value = "true")]
  139. stream: Option<bool>,
  140. #[arg(long, default_value_t = 64_000)]
  141. max_tokens: u32,
  142. #[arg(long)]
  143. debug_raw: bool,
  144. #[arg(long, num_args = 0..=1, default_missing_value = "true")]
  145. thinking: Option<bool>,
  146. #[arg(long)]
  147. thinking_type: Option<String>,
  148. #[arg(long)]
  149. thinking_budget_tokens: Option<u32>,
  150. #[arg(long)]
  151. thinking_effort: Option<String>,
  152. #[arg(long)]
  153. thinking_display: Option<String>,
  154. #[arg(long)]
  155. reasoning_effort: Option<String>,
  156. #[arg(long)]
  157. reasoning_summary: Option<String>,
  158. },
  159. GpqaDiamond {
  160. #[arg(long, default_value = "config.yaml")]
  161. config: PathBuf,
  162. #[arg(long)]
  163. provider: Option<String>,
  164. #[arg(long)]
  165. model: Option<String>,
  166. #[arg(long, default_value_t = 4)]
  167. concurrency: usize,
  168. #[arg(long)]
  169. limit: Option<usize>,
  170. #[arg(long, num_args = 0..=1, default_missing_value = "true")]
  171. stream: Option<bool>,
  172. #[arg(long, default_value_t = 64_000)]
  173. max_tokens: u32,
  174. #[arg(long)]
  175. debug_raw: bool,
  176. #[arg(long, num_args = 0..=1, default_missing_value = "true")]
  177. thinking: Option<bool>,
  178. #[arg(long)]
  179. thinking_type: Option<String>,
  180. #[arg(long)]
  181. thinking_budget_tokens: Option<u32>,
  182. #[arg(long)]
  183. thinking_effort: Option<String>,
  184. #[arg(long)]
  185. thinking_display: Option<String>,
  186. #[arg(long)]
  187. reasoning_effort: Option<String>,
  188. #[arg(long)]
  189. reasoning_summary: Option<String>,
  190. },
  191. }
  192. pub async fn dispatch(cli: Cli) -> Result<()> {
  193. match cli.command {
  194. Command::Check {
  195. config,
  196. provider,
  197. model,
  198. prompt,
  199. stream,
  200. debug_raw,
  201. thinking,
  202. thinking_type,
  203. thinking_budget_tokens,
  204. thinking_effort,
  205. thinking_display,
  206. reasoning_effort,
  207. reasoning_summary,
  208. } => {
  209. let config = AppConfig::load(&config)?;
  210. let provider_name = provider_name(&config, provider.as_deref())?;
  211. let provider = config.resolved_provider(Some(&provider_name))?;
  212. let model = model.unwrap_or_else(|| provider.default_model.clone());
  213. let request = ModelRequest {
  214. base_url: provider.base_url.clone(),
  215. api_token: provider.api_token.clone(),
  216. model: model.clone(),
  217. prompt,
  218. temperature: 0.0,
  219. max_tokens: 1024,
  220. stream: stream.unwrap_or(provider.stream),
  221. raw_debug: raw_debug_config(debug_raw, &provider_name, &model),
  222. thinking: merged_thinking_config(
  223. provider.thinking.as_ref(),
  224. ThinkingOverrides {
  225. thinking,
  226. thinking_type,
  227. thinking_budget_tokens,
  228. thinking_effort,
  229. thinking_display,
  230. reasoning_effort,
  231. reasoning_summary,
  232. },
  233. ),
  234. };
  235. let response = run_model_request(provider.protocol, request).await?;
  236. println!("status: {}", response.status);
  237. println!("elapsed_ms: {}", response.elapsed_ms);
  238. if let Some(ttft) = response.first_token_ms {
  239. println!("first_token_ms: {}", ttft);
  240. }
  241. println!("{}", response.text);
  242. Ok(())
  243. }
  244. Command::Dataset {
  245. command: DatasetCommand::Fetch { dataset },
  246. } => {
  247. let data_dir = dataset_data_dir(Path::new("config.yaml"))?;
  248. let path = benchmarks::fetch_dataset(&dataset, &data_dir).await?;
  249. println!("{}", path.display());
  250. Ok(())
  251. }
  252. Command::Bench { command } => dispatch_bench(command).await,
  253. Command::Rpm {
  254. config,
  255. mode,
  256. provider,
  257. model,
  258. rpm,
  259. duration,
  260. burst,
  261. probe_seconds,
  262. window_offset_ms,
  263. concurrency,
  264. prompt,
  265. stream,
  266. debug_raw,
  267. thinking,
  268. thinking_type,
  269. thinking_budget_tokens,
  270. thinking_effort,
  271. thinking_display,
  272. reasoning_effort,
  273. reasoning_summary,
  274. } => {
  275. run_rpm(
  276. config,
  277. RpmCommandOptions {
  278. mode,
  279. provider,
  280. model,
  281. rpm,
  282. duration,
  283. burst,
  284. probe_seconds,
  285. window_offset_ms,
  286. concurrency,
  287. prompt,
  288. stream,
  289. debug_raw,
  290. thinking,
  291. thinking_type,
  292. thinking_budget_tokens,
  293. thinking_effort,
  294. thinking_display,
  295. reasoning_effort,
  296. reasoning_summary,
  297. },
  298. )
  299. .await
  300. }
  301. }
  302. }
  303. async fn dispatch_bench(command: BenchCommand) -> Result<()> {
  304. match command {
  305. BenchCommand::Aime2026 {
  306. config,
  307. provider,
  308. model,
  309. concurrency,
  310. limit,
  311. stream,
  312. max_tokens,
  313. debug_raw,
  314. thinking,
  315. thinking_type,
  316. thinking_budget_tokens,
  317. thinking_effort,
  318. thinking_display,
  319. reasoning_effort,
  320. reasoning_summary,
  321. } => {
  322. run_aime_benchmark(BenchmarkCommandOptions {
  323. config,
  324. provider,
  325. model,
  326. concurrency,
  327. limit,
  328. stream,
  329. max_tokens,
  330. debug_raw,
  331. thinking_overrides: ThinkingOverrides {
  332. thinking,
  333. thinking_type,
  334. thinking_budget_tokens,
  335. thinking_effort,
  336. thinking_display,
  337. reasoning_effort,
  338. reasoning_summary,
  339. },
  340. })
  341. .await
  342. }
  343. BenchCommand::GpqaDiamond {
  344. config,
  345. provider,
  346. model,
  347. concurrency,
  348. limit,
  349. stream,
  350. max_tokens,
  351. debug_raw,
  352. thinking,
  353. thinking_type,
  354. thinking_budget_tokens,
  355. thinking_effort,
  356. thinking_display,
  357. reasoning_effort,
  358. reasoning_summary,
  359. } => {
  360. run_gpqa_benchmark(BenchmarkCommandOptions {
  361. config,
  362. provider,
  363. model,
  364. concurrency,
  365. limit,
  366. stream,
  367. max_tokens,
  368. debug_raw,
  369. thinking_overrides: ThinkingOverrides {
  370. thinking,
  371. thinking_type,
  372. thinking_budget_tokens,
  373. thinking_effort,
  374. thinking_display,
  375. reasoning_effort,
  376. reasoning_summary,
  377. },
  378. })
  379. .await
  380. }
  381. }
  382. }
  383. struct BenchmarkCommandOptions {
  384. config: PathBuf,
  385. provider: Option<String>,
  386. model: Option<String>,
  387. concurrency: usize,
  388. limit: Option<usize>,
  389. stream: Option<bool>,
  390. max_tokens: u32,
  391. debug_raw: bool,
  392. thinking_overrides: ThinkingOverrides,
  393. }
  394. async fn run_aime_benchmark(options: BenchmarkCommandOptions) -> Result<()> {
  395. let config = AppConfig::load(&options.config)?;
  396. let provider_name = provider_name(&config, options.provider.as_deref())?;
  397. let provider_config = config.resolved_provider(Some(&provider_name))?;
  398. let model = options
  399. .model
  400. .unwrap_or_else(|| provider_config.default_model.clone());
  401. let loaded = benchmarks::aime::load_cases(Path::new(&config.benchmarks.data_dir))?;
  402. let dataset = dataset_report(
  403. config
  404. .benchmarks
  405. .aime2026
  406. .as_ref()
  407. .map(|dataset| (dataset.source.as_str(), dataset.split.as_str())),
  408. &loaded.local_path,
  409. );
  410. let cases = apply_limit(loaded.cases, options.limit);
  411. let total = cases.len() as u64;
  412. let started_at = Utc::now();
  413. let started = Instant::now();
  414. let mut base_request = request_template(&provider_config, &model, 0.0, options.max_tokens);
  415. base_request.stream = options.stream.unwrap_or(provider_config.stream);
  416. base_request.raw_debug = raw_debug_config(options.debug_raw, &provider_name, &model);
  417. base_request.thinking = merged_thinking_config(
  418. provider_config.thinking.as_ref(),
  419. options.thinking_overrides,
  420. );
  421. let protocol = provider_config.protocol;
  422. let pb = ProgressBar::new(total);
  423. pb.set_style(
  424. ProgressStyle::default_bar()
  425. .template("[{elapsed_precise}] {bar:40} {pos}/{len} ({eta})")
  426. .expect("valid template"),
  427. );
  428. let mut results = stream::iter(cases)
  429. .map(|case| {
  430. let mut request = base_request.clone();
  431. request.prompt = case.prompt();
  432. async move {
  433. let result = run_model_request(protocol, request).await;
  434. (case, result)
  435. }
  436. })
  437. .buffer_unordered(nonzero_concurrency(options.concurrency));
  438. let mut metrics = Metrics::new();
  439. let mut wrong_cases = Vec::new();
  440. let mut correct_samples = Vec::new();
  441. let mut debug_requests = Vec::new();
  442. while let Some((case, result)) = results.next().await {
  443. pb.inc(1);
  444. match result {
  445. Ok(response) => {
  446. let prompt = case.prompt();
  447. if let Some(debug) =
  448. response_debug_report(Some(case.id.clone()), None, None, prompt, &response)
  449. {
  450. debug_requests.push(debug);
  451. }
  452. if response_has_no_content(&response) {
  453. metrics.record_failure("request_error");
  454. continue;
  455. }
  456. metrics.record_success(
  457. response.status,
  458. response.elapsed_ms as u64,
  459. response.first_token_ms.map(|ms| ms as u64),
  460. );
  461. let actual = judge::extract_final_integer(&response.text)
  462. .unwrap_or_else(|| "no_answer".to_string());
  463. let correct = judge::judge_integer(&response.text, &case.answer);
  464. metrics.record_judgement(correct);
  465. if correct {
  466. if correct_samples.len() < 5 {
  467. correct_samples.push(CorrectCaseReport {
  468. id: case.id,
  469. question: case.problem,
  470. expected: case.answer,
  471. raw_output: response.text,
  472. });
  473. }
  474. } else {
  475. wrong_cases.push(WrongCaseReport {
  476. id: case.id,
  477. question: case.problem,
  478. expected: case.answer,
  479. actual,
  480. raw_output: response.text,
  481. });
  482. }
  483. }
  484. Err(error) => metrics.record_failure(error_code(&error)),
  485. }
  486. }
  487. pb.finish_and_clear();
  488. let summary = metrics.summary();
  489. let report = benchmark_report(BenchmarkReportInput {
  490. benchmark: "aime2026",
  491. provider: provider_name,
  492. model,
  493. stream: base_request.stream,
  494. thinking: thinking_report(base_request.thinking.as_ref()),
  495. request: sent_request_params(protocol, &base_request),
  496. dataset,
  497. started_at,
  498. duration_ms: started.elapsed().as_millis(),
  499. concurrency: options.concurrency,
  500. limit: options.limit,
  501. max_tokens: options.max_tokens,
  502. summary,
  503. correct_samples,
  504. wrong_cases,
  505. debug_requests,
  506. });
  507. let report_path = write_benchmark_report(Path::new("."), &report)?;
  508. print_benchmark_report(&report, &report_path);
  509. Ok(())
  510. }
  511. async fn run_gpqa_benchmark(options: BenchmarkCommandOptions) -> Result<()> {
  512. let config = AppConfig::load(&options.config)?;
  513. let provider_name = provider_name(&config, options.provider.as_deref())?;
  514. let provider_config = config.resolved_provider(Some(&provider_name))?;
  515. let model = options
  516. .model
  517. .unwrap_or_else(|| provider_config.default_model.clone());
  518. let loaded = benchmarks::gpqa::load_cases(Path::new(&config.benchmarks.data_dir))?;
  519. let dataset = dataset_report(
  520. config
  521. .benchmarks
  522. .gpqa_diamond
  523. .as_ref()
  524. .map(|dataset| (dataset.source.as_str(), dataset.split.as_str())),
  525. &loaded.local_path,
  526. );
  527. let cases = apply_limit(loaded.cases, options.limit);
  528. let total = cases.len() as u64;
  529. let started_at = Utc::now();
  530. let started = Instant::now();
  531. let mut base_request = request_template(&provider_config, &model, 0.0, options.max_tokens);
  532. base_request.stream = options.stream.unwrap_or(provider_config.stream);
  533. base_request.raw_debug = raw_debug_config(options.debug_raw, &provider_name, &model);
  534. base_request.thinking = merged_thinking_config(
  535. provider_config.thinking.as_ref(),
  536. options.thinking_overrides,
  537. );
  538. let protocol = provider_config.protocol;
  539. let pb = ProgressBar::new(total);
  540. pb.set_style(
  541. ProgressStyle::default_bar()
  542. .template("[{elapsed_precise}] {bar:40} {pos}/{len} ({eta})")
  543. .expect("valid template"),
  544. );
  545. let mut results = stream::iter(cases)
  546. .map(|case| {
  547. let mut request = base_request.clone();
  548. request.prompt = case.prompt();
  549. async move {
  550. let result = run_model_request(protocol, request).await;
  551. (case, result)
  552. }
  553. })
  554. .buffer_unordered(nonzero_concurrency(options.concurrency));
  555. let mut metrics = Metrics::new();
  556. let mut wrong_cases = Vec::new();
  557. let mut correct_samples = Vec::new();
  558. let mut debug_requests = Vec::new();
  559. while let Some((case, result)) = results.next().await {
  560. pb.inc(1);
  561. match result {
  562. Ok(response) => {
  563. let prompt = case.prompt();
  564. if let Some(debug) =
  565. response_debug_report(Some(case.id.clone()), None, None, prompt, &response)
  566. {
  567. debug_requests.push(debug);
  568. }
  569. if response_has_no_content(&response) {
  570. metrics.record_failure("request_error");
  571. continue;
  572. }
  573. metrics.record_success(
  574. response.status,
  575. response.elapsed_ms as u64,
  576. response.first_token_ms.map(|ms| ms as u64),
  577. );
  578. let actual = judge::extract_choice(&response.text)
  579. .map(|choice| choice.to_string())
  580. .unwrap_or_else(|| "no_answer".to_string());
  581. let expected = case.answer.to_string();
  582. let correct = judge::judge_choice(&response.text, case.answer);
  583. metrics.record_judgement(correct);
  584. if correct {
  585. if correct_samples.len() < 5 {
  586. correct_samples.push(CorrectCaseReport {
  587. id: case.id,
  588. question: case.question,
  589. expected,
  590. raw_output: response.text,
  591. });
  592. }
  593. } else {
  594. wrong_cases.push(WrongCaseReport {
  595. id: case.id,
  596. question: case.question,
  597. expected,
  598. actual,
  599. raw_output: response.text,
  600. });
  601. }
  602. }
  603. Err(error) => metrics.record_failure(error_code(&error)),
  604. }
  605. }
  606. pb.finish_and_clear();
  607. let summary = metrics.summary();
  608. let report = benchmark_report(BenchmarkReportInput {
  609. benchmark: "gpqa-diamond",
  610. provider: provider_name,
  611. model,
  612. stream: base_request.stream,
  613. thinking: thinking_report(base_request.thinking.as_ref()),
  614. request: sent_request_params(protocol, &base_request),
  615. dataset,
  616. started_at,
  617. duration_ms: started.elapsed().as_millis(),
  618. concurrency: options.concurrency,
  619. limit: options.limit,
  620. max_tokens: options.max_tokens,
  621. summary,
  622. correct_samples,
  623. wrong_cases,
  624. debug_requests,
  625. });
  626. let report_path = write_benchmark_report(Path::new("."), &report)?;
  627. print_benchmark_report(&report, &report_path);
  628. Ok(())
  629. }
  630. struct RpmCommandOptions {
  631. mode: RpmMode,
  632. provider: Option<String>,
  633. model: Option<String>,
  634. rpm: Option<u32>,
  635. duration: String,
  636. burst: Option<u32>,
  637. probe_seconds: Option<u64>,
  638. window_offset_ms: u64,
  639. concurrency: Option<usize>,
  640. prompt: String,
  641. stream: Option<bool>,
  642. debug_raw: bool,
  643. thinking: Option<bool>,
  644. thinking_type: Option<String>,
  645. thinking_budget_tokens: Option<u32>,
  646. thinking_effort: Option<String>,
  647. thinking_display: Option<String>,
  648. reasoning_effort: Option<String>,
  649. reasoning_summary: Option<String>,
  650. }
  651. async fn run_rpm(config_path: PathBuf, options: RpmCommandOptions) -> Result<()> {
  652. let config = AppConfig::load(&config_path)?;
  653. let mode_plan = build_rpm_mode_plan(
  654. options.mode,
  655. options.rpm,
  656. &options.duration,
  657. options.burst,
  658. options.probe_seconds,
  659. options.window_offset_ms,
  660. )?;
  661. let provider_name = provider_name(&config, options.provider.as_deref())?;
  662. let provider_config = config.resolved_provider(Some(&provider_name))?;
  663. let model = options
  664. .model
  665. .unwrap_or_else(|| provider_config.default_model.clone());
  666. let stream_enabled = options.stream.unwrap_or(provider_config.stream);
  667. let concurrency = options.concurrency.unwrap_or(mode_plan.default_concurrency);
  668. let request = ModelRequest {
  669. prompt: options.prompt.clone(),
  670. stream: stream_enabled,
  671. raw_debug: raw_debug_config(options.debug_raw, &provider_name, &model),
  672. thinking: merged_thinking_config(
  673. provider_config.thinking.as_ref(),
  674. ThinkingOverrides {
  675. thinking: options.thinking,
  676. thinking_type: options.thinking_type,
  677. thinking_budget_tokens: options.thinking_budget_tokens,
  678. thinking_effort: options.thinking_effort,
  679. thinking_display: options.thinking_display,
  680. reasoning_effort: options.reasoning_effort,
  681. reasoning_summary: options.reasoning_summary,
  682. },
  683. ),
  684. ..request_template(&provider_config, &model, 0.0, 1024)
  685. };
  686. let thinking = thinking_report(request.thinking.as_ref());
  687. let started_at = Utc::now();
  688. let started = Instant::now();
  689. let mut metrics = Metrics::new();
  690. let mut mode_summary = RpmModeSummaryBuilder::default();
  691. let mut debug_requests = Vec::new();
  692. let prompt_for_debug = request.prompt.clone();
  693. let results = run_scheduled_requests(
  694. provider_config.protocol,
  695. request,
  696. mode_plan.probes,
  697. concurrency,
  698. )
  699. .await;
  700. for result in results {
  701. let success = result
  702. .result
  703. .as_ref()
  704. .is_ok_and(|response| !response_has_no_content(response));
  705. mode_summary.record(result.phase, result.second, success);
  706. match result.result {
  707. Ok(response) => {
  708. if let Some(debug) = response_debug_report(
  709. None,
  710. Some(phase_name(result.phase).to_string()),
  711. result.second,
  712. prompt_for_debug.clone(),
  713. &response,
  714. ) {
  715. debug_requests.push(debug);
  716. }
  717. if response_has_no_content(&response) {
  718. metrics.record_failure("request_error");
  719. continue;
  720. }
  721. metrics.record_success(
  722. response.status,
  723. response.elapsed_ms as u64,
  724. response.first_token_ms.map(|ms| ms as u64),
  725. );
  726. }
  727. Err(error) => metrics.record_failure(error_code(&error)),
  728. }
  729. }
  730. let summary = metrics.summary();
  731. let report = RpmReport {
  732. benchmark: "rpm".to_string(),
  733. provider: provider_name,
  734. model,
  735. params: RpmParamsReport {
  736. prompt: options.prompt,
  737. stream: stream_enabled,
  738. thinking,
  739. duration: options.duration,
  740. burst: options.burst,
  741. concurrency,
  742. probe_seconds: options.probe_seconds,
  743. window_offset_ms: options.window_offset_ms,
  744. },
  745. run: RpmRunReport {
  746. started_at,
  747. duration_ms: started.elapsed().as_millis(),
  748. target_rpm: mode_plan.target_rpm,
  749. actual_rpm: actual_rpm(summary.total, started.elapsed()),
  750. temperature: 0.0,
  751. max_tokens: 1024,
  752. },
  753. summary: RpmSummaryReport {
  754. actual_requests: summary.total,
  755. success: summary.success,
  756. failure: summary.failed,
  757. latency_ms: latency_report(&summary.latency_ms),
  758. ttft_ms: latency_report(&summary.ttft_ms),
  759. },
  760. mode: mode_plan.mode_name.to_string(),
  761. mode_detail: mode_summary.into_report(options.mode),
  762. errors: summary.errors,
  763. debug_requests,
  764. };
  765. let report_path = write_rpm_report(Path::new("."), &report)?;
  766. print_rpm_report(&report, &report_path);
  767. Ok(())
  768. }
  769. #[derive(Debug)]
  770. struct RpmModePlan {
  771. mode_name: &'static str,
  772. target_rpm: u32,
  773. probes: Vec<ScheduledProbe>,
  774. default_concurrency: usize,
  775. }
  776. #[derive(Debug)]
  777. struct ScheduledResult {
  778. phase: ProbePhase,
  779. second: Option<u64>,
  780. result: Result<crate::runner::ModelResponse>,
  781. }
  782. fn build_rpm_mode_plan(
  783. mode: RpmMode,
  784. rpm: Option<u32>,
  785. duration: &str,
  786. burst: Option<u32>,
  787. probe_seconds: Option<u64>,
  788. window_offset_ms: u64,
  789. ) -> Result<RpmModePlan> {
  790. let target_rpm = rpm.unwrap_or(0);
  791. let burst = burst.unwrap_or(target_rpm);
  792. let probes = match mode {
  793. RpmMode::Sustained => {
  794. let rpm = require_positive("rpm", rpm)?;
  795. let duration = parse_duration(duration)?;
  796. sustained_schedule(duration, rpm)
  797. .into_iter()
  798. .map(|offset| ScheduledProbe {
  799. offset,
  800. phase: ProbePhase::RefillProbe,
  801. second: Some(offset.as_secs()),
  802. })
  803. .collect()
  804. }
  805. RpmMode::Burst => {
  806. let burst = require_positive_value("burst", burst)?;
  807. burst_schedule(burst)
  808. .into_iter()
  809. .map(|offset| ScheduledProbe {
  810. offset,
  811. phase: ProbePhase::Burst,
  812. second: Some(0),
  813. })
  814. .collect()
  815. }
  816. RpmMode::TokenBucket => {
  817. let rpm = require_positive("rpm", rpm)?;
  818. let burst = require_positive_value("burst", burst)?;
  819. token_bucket_schedule(rpm, burst, probe_seconds.unwrap_or(30))
  820. }
  821. RpmMode::SlidingWindow => {
  822. let burst = require_positive_value("burst", burst)?;
  823. sliding_window_schedule(burst, probe_seconds.unwrap_or(90))
  824. }
  825. RpmMode::WindowBoundary => {
  826. let burst = require_positive_value("burst", burst)?;
  827. window_boundary_plan(Utc::now(), burst, window_offset_ms).probes
  828. }
  829. RpmMode::Diagnose => {
  830. let rpm = require_positive("rpm", rpm)?;
  831. let burst = require_positive_value("burst", burst)?;
  832. let mut probes = token_bucket_schedule(rpm, burst, probe_seconds.unwrap_or(90));
  833. probes.extend(sliding_window_schedule(burst, probe_seconds.unwrap_or(90)));
  834. probes.extend(window_boundary_plan(Utc::now(), burst, window_offset_ms).probes);
  835. probes
  836. }
  837. };
  838. let default_concurrency = probes.len().max(1);
  839. Ok(RpmModePlan {
  840. mode_name: mode_name(mode),
  841. target_rpm,
  842. default_concurrency,
  843. probes,
  844. })
  845. }
  846. fn mode_name(mode: RpmMode) -> &'static str {
  847. match mode {
  848. RpmMode::Sustained => "sustained",
  849. RpmMode::Burst => "burst",
  850. RpmMode::TokenBucket => "token-bucket",
  851. RpmMode::SlidingWindow => "sliding-window",
  852. RpmMode::WindowBoundary => "window-boundary",
  853. RpmMode::Diagnose => "diagnose",
  854. }
  855. }
  856. #[derive(Default)]
  857. struct RpmModeSummaryBuilder {
  858. phases: BTreeMap<&'static str, PhaseAccumulator>,
  859. refill_seconds: BTreeMap<u64, PhaseAccumulator>,
  860. sliding_seconds: BTreeMap<u64, PhaseAccumulator>,
  861. }
  862. impl RpmModeSummaryBuilder {
  863. fn record(&mut self, phase: ProbePhase, second: Option<u64>, success: bool) {
  864. match phase {
  865. ProbePhase::Burst => self.phase("burst").record(success),
  866. ProbePhase::RefillProbe => {
  867. self.phase("refill_probe").record(success);
  868. self.refill_seconds
  869. .entry(second.unwrap_or(0))
  870. .or_default()
  871. .record(success);
  872. }
  873. ProbePhase::SlidingProbe => {
  874. self.phase("sliding_probe").record(success);
  875. self.sliding_seconds
  876. .entry(second.unwrap_or(0))
  877. .or_default()
  878. .record(success);
  879. }
  880. ProbePhase::BeforeBoundary => self.phase("before_boundary").record(success),
  881. ProbePhase::AfterBoundary => self.phase("after_boundary").record(success),
  882. }
  883. }
  884. fn into_report(self, mode: RpmMode) -> Option<RpmModeDetailReport> {
  885. if mode == RpmMode::Sustained {
  886. return None;
  887. }
  888. let burst = self.phases.get("burst").map(PhaseAccumulator::to_report);
  889. let refill_probe = probe_seconds_report(self.refill_seconds);
  890. let sliding_probe = probe_seconds_report(self.sliding_seconds);
  891. let window_boundary = match (
  892. self.phases.get("before_boundary"),
  893. self.phases.get("after_boundary"),
  894. ) {
  895. (Some(before), Some(after)) => Some(WindowBoundaryReport {
  896. before: before.to_report(),
  897. after: after.to_report(),
  898. }),
  899. _ => None,
  900. };
  901. let inference = if mode == RpmMode::Diagnose {
  902. Some(infer_limiter(
  903. burst.as_ref(),
  904. &refill_probe,
  905. &sliding_probe,
  906. window_boundary.as_ref(),
  907. ))
  908. } else {
  909. None
  910. };
  911. Some(RpmModeDetailReport {
  912. burst,
  913. refill_probe,
  914. sliding_probe,
  915. window_boundary,
  916. inference,
  917. })
  918. }
  919. fn phase(&mut self, name: &'static str) -> &mut PhaseAccumulator {
  920. self.phases.entry(name).or_default()
  921. }
  922. }
  923. #[derive(Default)]
  924. struct PhaseAccumulator {
  925. sent: u64,
  926. success: u64,
  927. failure: u64,
  928. }
  929. impl PhaseAccumulator {
  930. fn record(&mut self, success: bool) {
  931. self.sent += 1;
  932. if success {
  933. self.success += 1;
  934. } else {
  935. self.failure += 1;
  936. }
  937. }
  938. fn to_report(&self) -> PhaseSummaryReport {
  939. PhaseSummaryReport {
  940. sent: self.sent,
  941. success: self.success,
  942. failure: self.failure,
  943. }
  944. }
  945. fn success_rate(&self) -> f64 {
  946. if self.sent == 0 {
  947. 0.0
  948. } else {
  949. self.success as f64 / self.sent as f64
  950. }
  951. }
  952. }
  953. fn probe_seconds_report(seconds: BTreeMap<u64, PhaseAccumulator>) -> Vec<ProbeSecondReport> {
  954. seconds
  955. .into_iter()
  956. .map(|(second, accumulator)| ProbeSecondReport {
  957. second,
  958. sent: accumulator.sent,
  959. success: accumulator.success,
  960. failure: accumulator.failure,
  961. })
  962. .collect()
  963. }
  964. fn infer_limiter(
  965. _burst: Option<&PhaseSummaryReport>,
  966. refill_probe: &[ProbeSecondReport],
  967. sliding_probe: &[ProbeSecondReport],
  968. window_boundary: Option<&WindowBoundaryReport>,
  969. ) -> LimiterInferenceReport {
  970. let mut signals = Vec::new();
  971. if let Some(boundary) = window_boundary {
  972. let before_rate = phase_success_rate(&boundary.before);
  973. let after_rate = phase_success_rate(&boundary.after);
  974. if after_rate > before_rate + 0.3 {
  975. signals.push("after-boundary success rate was much higher than before-boundary".into());
  976. return LimiterInferenceReport {
  977. likely_limiter: LimiterInferenceKind::FixedWindow,
  978. confidence: "medium".to_string(),
  979. signals,
  980. };
  981. }
  982. }
  983. let refill_sent: u64 = refill_probe.iter().map(|probe| probe.sent).sum();
  984. let refill_success: u64 = refill_probe.iter().map(|probe| probe.success).sum();
  985. if refill_sent > 0 && refill_success as f64 / refill_sent as f64 >= 0.5 {
  986. signals.push("refill probes recovered at a steady rate".into());
  987. return LimiterInferenceReport {
  988. likely_limiter: LimiterInferenceKind::TokenBucket,
  989. confidence: "medium".to_string(),
  990. signals,
  991. };
  992. }
  993. let early = sliding_probe
  994. .iter()
  995. .filter(|probe| probe.second <= 30)
  996. .fold(PhaseAccumulator::default(), |mut acc, probe| {
  997. acc.sent += probe.sent;
  998. acc.success += probe.success;
  999. acc.failure += probe.failure;
  1000. acc
  1001. });
  1002. let late = sliding_probe
  1003. .iter()
  1004. .filter(|probe| probe.second >= 60)
  1005. .fold(PhaseAccumulator::default(), |mut acc, probe| {
  1006. acc.sent += probe.sent;
  1007. acc.success += probe.success;
  1008. acc.failure += probe.failure;
  1009. acc
  1010. });
  1011. if late.sent > 0 && late.success_rate() > early.success_rate() + 0.3 {
  1012. signals.push("probe recovery improved near the 60 second rolling window".into());
  1013. return LimiterInferenceReport {
  1014. likely_limiter: LimiterInferenceKind::SlidingWindow,
  1015. confidence: "medium".to_string(),
  1016. signals,
  1017. };
  1018. }
  1019. signals.push("signals did not clearly match a limiter model".into());
  1020. LimiterInferenceReport {
  1021. likely_limiter: LimiterInferenceKind::Unknown,
  1022. confidence: "low".to_string(),
  1023. signals,
  1024. }
  1025. }
  1026. fn phase_success_rate(phase: &PhaseSummaryReport) -> f64 {
  1027. if phase.sent == 0 {
  1028. 0.0
  1029. } else {
  1030. phase.success as f64 / phase.sent as f64
  1031. }
  1032. }
  1033. fn actual_rpm(total_requests: u64, elapsed: Duration) -> Option<f64> {
  1034. let elapsed_seconds = elapsed.as_secs_f64();
  1035. if elapsed_seconds == 0.0 {
  1036. None
  1037. } else {
  1038. Some(total_requests as f64 / elapsed_seconds * 60.0)
  1039. }
  1040. }
  1041. fn require_positive(name: &str, value: Option<u32>) -> Result<u32> {
  1042. let value = value.with_context(|| format!("{name} is required for this rpm mode"))?;
  1043. require_positive_value(name, value)
  1044. }
  1045. fn require_positive_value(name: &str, value: u32) -> Result<u32> {
  1046. if value == 0 {
  1047. bail!("{name} must be greater than 0");
  1048. }
  1049. Ok(value)
  1050. }
  1051. async fn run_scheduled_requests(
  1052. protocol: crate::config::ProtocolKind,
  1053. request: ModelRequest,
  1054. starts: Vec<ScheduledProbe>,
  1055. max_in_flight: usize,
  1056. ) -> Vec<ScheduledResult> {
  1057. let total = starts.len() as u64;
  1058. let pb = ProgressBar::new(total);
  1059. pb.set_style(
  1060. ProgressStyle::default_bar()
  1061. .template("[{elapsed_precise}] {bar:40} {pos}/{len} ({eta})")
  1062. .expect("valid template"),
  1063. );
  1064. let tokio_started = TokioInstant::now();
  1065. let mut results = Vec::with_capacity(starts.len());
  1066. let mut s = stream::iter(starts.into_iter().map(|start| {
  1067. let request = request.clone();
  1068. async move {
  1069. sleep_until(tokio_started + start.offset).await;
  1070. ScheduledResult {
  1071. phase: start.phase,
  1072. second: start.second,
  1073. result: run_model_request(protocol, request).await,
  1074. }
  1075. }
  1076. }))
  1077. .buffer_unordered(nonzero_concurrency(max_in_flight));
  1078. while let Some(result) = s.next().await {
  1079. pb.inc(1);
  1080. results.push(result);
  1081. }
  1082. pb.finish_and_clear();
  1083. results
  1084. }
  1085. fn provider_name(config: &AppConfig, provider: Option<&str>) -> Result<String> {
  1086. match provider {
  1087. Some(provider) => Ok(provider.to_string()),
  1088. None => config
  1089. .default_provider
  1090. .clone()
  1091. .context("no provider specified and config has no default_provider"),
  1092. }
  1093. }
  1094. fn raw_debug_config(enabled: bool, provider: &str, model: &str) -> Option<RawDebugConfig> {
  1095. Some(
  1096. RawDebugConfig::new(
  1097. PathBuf::from("outputs/debug"),
  1098. format!("{provider}-{model}"),
  1099. )
  1100. .with_success_raw(enabled),
  1101. )
  1102. }
  1103. #[derive(Debug, Default)]
  1104. struct ThinkingOverrides {
  1105. thinking: Option<bool>,
  1106. thinking_type: Option<String>,
  1107. thinking_budget_tokens: Option<u32>,
  1108. thinking_effort: Option<String>,
  1109. thinking_display: Option<String>,
  1110. reasoning_effort: Option<String>,
  1111. reasoning_summary: Option<String>,
  1112. }
  1113. fn merged_thinking_config(
  1114. provider: Option<&ProviderThinkingConfig>,
  1115. overrides: ThinkingOverrides,
  1116. ) -> Option<ThinkingConfig> {
  1117. let has_overrides = overrides.thinking.is_some()
  1118. || overrides.thinking_type.is_some()
  1119. || overrides.thinking_budget_tokens.is_some()
  1120. || overrides.thinking_effort.is_some()
  1121. || overrides.thinking_display.is_some()
  1122. || overrides.reasoning_effort.is_some()
  1123. || overrides.reasoning_summary.is_some();
  1124. if provider.is_none() && !has_overrides {
  1125. return None;
  1126. }
  1127. let enabled = overrides
  1128. .thinking
  1129. .or_else(|| provider.and_then(|thinking| thinking.enabled))
  1130. .unwrap_or(true);
  1131. if !enabled {
  1132. return Some(ThinkingConfig {
  1133. enabled: false,
  1134. kind: overrides
  1135. .thinking_type
  1136. .or_else(|| provider.and_then(|thinking| thinking.kind.clone())),
  1137. budget_tokens: overrides
  1138. .thinking_budget_tokens
  1139. .or_else(|| provider.and_then(|thinking| thinking.budget_tokens)),
  1140. effort: overrides
  1141. .thinking_effort
  1142. .or_else(|| provider.and_then(|thinking| thinking.effort.clone())),
  1143. display: overrides
  1144. .thinking_display
  1145. .or_else(|| provider.and_then(|thinking| thinking.display.clone())),
  1146. reasoning_effort: overrides
  1147. .reasoning_effort
  1148. .or_else(|| provider.and_then(|thinking| thinking.reasoning_effort.clone())),
  1149. reasoning_summary: overrides
  1150. .reasoning_summary
  1151. .or_else(|| provider.and_then(|thinking| thinking.reasoning_summary.clone())),
  1152. });
  1153. }
  1154. Some(ThinkingConfig {
  1155. enabled,
  1156. kind: overrides
  1157. .thinking_type
  1158. .or_else(|| provider.and_then(|thinking| thinking.kind.clone()))
  1159. .or_else(|| Some("enabled".to_string())),
  1160. budget_tokens: overrides
  1161. .thinking_budget_tokens
  1162. .or_else(|| provider.and_then(|thinking| thinking.budget_tokens)),
  1163. effort: overrides
  1164. .thinking_effort
  1165. .or_else(|| provider.and_then(|thinking| thinking.effort.clone())),
  1166. display: overrides
  1167. .thinking_display
  1168. .or_else(|| provider.and_then(|thinking| thinking.display.clone())),
  1169. reasoning_effort: overrides
  1170. .reasoning_effort
  1171. .or_else(|| provider.and_then(|thinking| thinking.reasoning_effort.clone())),
  1172. reasoning_summary: overrides
  1173. .reasoning_summary
  1174. .or_else(|| provider.and_then(|thinking| thinking.reasoning_summary.clone())),
  1175. })
  1176. }
  1177. fn request_template(
  1178. provider: &crate::config::ProviderConfig,
  1179. model: &str,
  1180. temperature: f32,
  1181. max_tokens: u32,
  1182. ) -> ModelRequest {
  1183. ModelRequest {
  1184. base_url: provider.base_url.clone(),
  1185. api_token: provider.api_token.clone(),
  1186. model: model.to_string(),
  1187. prompt: String::new(),
  1188. temperature,
  1189. max_tokens,
  1190. stream: provider.stream,
  1191. raw_debug: None,
  1192. thinking: merged_thinking_config(provider.thinking.as_ref(), ThinkingOverrides::default()),
  1193. }
  1194. }
  1195. fn apply_limit<T>(cases: Vec<T>, limit: Option<usize>) -> Vec<T> {
  1196. match limit {
  1197. Some(limit) => cases.into_iter().take(limit).collect(),
  1198. None => cases,
  1199. }
  1200. }
  1201. fn dataset_report(config: Option<(&str, &str)>, local_path: &Path) -> DatasetReport {
  1202. let (source, split) = config.unwrap_or(("local", "train"));
  1203. DatasetReport {
  1204. source: source.to_string(),
  1205. split: split.to_string(),
  1206. revision: None,
  1207. local_path: local_path.display().to_string(),
  1208. }
  1209. }
  1210. fn thinking_report(thinking: Option<&ThinkingConfig>) -> Option<ThinkingParamsReport> {
  1211. thinking.map(|thinking| ThinkingParamsReport {
  1212. enabled: thinking.enabled,
  1213. kind: thinking.kind.clone(),
  1214. budget_tokens: thinking.budget_tokens,
  1215. effort: thinking.effort.clone(),
  1216. display: thinking.display.clone(),
  1217. reasoning_effort: thinking.reasoning_effort.clone(),
  1218. reasoning_summary: thinking.reasoning_summary.clone(),
  1219. })
  1220. }
  1221. struct BenchmarkReportInput {
  1222. benchmark: &'static str,
  1223. provider: String,
  1224. model: String,
  1225. stream: bool,
  1226. thinking: Option<ThinkingParamsReport>,
  1227. request: SentRequestParamsReport,
  1228. dataset: DatasetReport,
  1229. started_at: chrono::DateTime<Utc>,
  1230. duration_ms: u128,
  1231. concurrency: usize,
  1232. limit: Option<usize>,
  1233. max_tokens: u32,
  1234. summary: MetricsSummary,
  1235. correct_samples: Vec<CorrectCaseReport>,
  1236. wrong_cases: Vec<WrongCaseReport>,
  1237. debug_requests: Vec<RequestDebugReport>,
  1238. }
  1239. fn benchmark_report(input: BenchmarkReportInput) -> BenchmarkReport {
  1240. BenchmarkReport {
  1241. benchmark: input.benchmark.to_string(),
  1242. provider: input.provider,
  1243. model: input.model,
  1244. params: BenchmarkParamsReport {
  1245. stream: input.stream,
  1246. thinking: input.thinking,
  1247. request: input.request,
  1248. },
  1249. dataset: input.dataset,
  1250. run: RunReport {
  1251. started_at: input.started_at,
  1252. duration_ms: input.duration_ms,
  1253. concurrency: input.concurrency,
  1254. limit: input.limit,
  1255. temperature: 0.0,
  1256. max_tokens: input.max_tokens,
  1257. },
  1258. summary: BenchmarkSummaryReport {
  1259. accuracy: input.summary.accuracy,
  1260. success: input.summary.success,
  1261. total: input.summary.total,
  1262. correct: input.summary.correct,
  1263. wrong: input.summary.wrong,
  1264. failed: input.summary.failed,
  1265. latency_ms: latency_report(&input.summary.latency_ms),
  1266. ttft_ms: latency_report(&input.summary.ttft_ms),
  1267. },
  1268. errors: input.summary.errors,
  1269. correct_samples: input.correct_samples,
  1270. wrong_cases: input.wrong_cases,
  1271. debug_requests: input.debug_requests,
  1272. }
  1273. }
  1274. const NEAR_TTFT_LATENCY_THRESHOLD_MS: u128 = 1_000;
  1275. fn response_debug_report(
  1276. id: Option<String>,
  1277. phase: Option<String>,
  1278. second: Option<u64>,
  1279. prompt: String,
  1280. response: &ModelResponse,
  1281. ) -> Option<RequestDebugReport> {
  1282. let ttft_latency_delta_ms = response
  1283. .first_token_ms
  1284. .map(|ttft_ms| response.elapsed_ms.abs_diff(ttft_ms));
  1285. let near_ttft_latency =
  1286. ttft_latency_delta_ms.is_some_and(|delta_ms| delta_ms < NEAR_TTFT_LATENCY_THRESHOLD_MS);
  1287. let has_think_tags = contains_think_tags(&response.text);
  1288. let no_content = response_has_no_content(response);
  1289. let mut reasons = Vec::new();
  1290. if no_content {
  1291. reasons.push("no_content");
  1292. }
  1293. if near_ttft_latency {
  1294. reasons.push("near_ttft_latency");
  1295. }
  1296. if has_think_tags {
  1297. reasons.push("contains_think_tags");
  1298. }
  1299. if reasons.is_empty() {
  1300. return None;
  1301. }
  1302. Some(RequestDebugReport {
  1303. reason: reasons.join(","),
  1304. id,
  1305. phase,
  1306. second,
  1307. prompt,
  1308. output: response.text.clone(),
  1309. latency_ms: response.elapsed_ms,
  1310. ttft_ms: response.first_token_ms,
  1311. ttft_latency_delta_ms,
  1312. has_think_tags,
  1313. })
  1314. }
  1315. fn response_has_no_content(response: &ModelResponse) -> bool {
  1316. response.text.trim().is_empty()
  1317. }
  1318. fn contains_think_tags(text: &str) -> bool {
  1319. let lower = text.to_ascii_lowercase();
  1320. lower.contains("<think>") && lower.contains("</think>")
  1321. }
  1322. fn phase_name(phase: ProbePhase) -> &'static str {
  1323. match phase {
  1324. ProbePhase::Burst => "burst",
  1325. ProbePhase::RefillProbe => "refill_probe",
  1326. ProbePhase::SlidingProbe => "sliding_probe",
  1327. ProbePhase::BeforeBoundary => "before_boundary",
  1328. ProbePhase::AfterBoundary => "after_boundary",
  1329. }
  1330. }
  1331. fn sent_request_params(protocol: ProtocolKind, request: &ModelRequest) -> SentRequestParamsReport {
  1332. let protocol_name = match protocol {
  1333. ProtocolKind::Openai => "openai",
  1334. ProtocolKind::Anthropic => "anthropic",
  1335. ProtocolKind::Google => "google",
  1336. };
  1337. SentRequestParamsReport {
  1338. protocol: protocol_name.to_string(),
  1339. body: sent_request_body_params(protocol, request),
  1340. }
  1341. }
  1342. fn sent_request_body_params(protocol: ProtocolKind, request: &ModelRequest) -> Value {
  1343. match protocol {
  1344. ProtocolKind::Openai => {
  1345. let mut body = serde_json::Map::new();
  1346. body.insert("temperature".to_string(), json!(request.temperature));
  1347. body.insert("max_tokens".to_string(), json!(request.max_tokens));
  1348. if request.stream {
  1349. body.insert("stream".to_string(), json!(true));
  1350. }
  1351. if let Some(thinking) = &request.thinking
  1352. && thinking.enabled
  1353. {
  1354. if let Some(reasoning_effort) = thinking
  1355. .reasoning_effort
  1356. .as_ref()
  1357. .or(thinking.effort.as_ref())
  1358. {
  1359. body.insert("reasoning_effort".to_string(), json!(reasoning_effort));
  1360. }
  1361. if let Some(reasoning_summary) = &thinking.reasoning_summary {
  1362. body.insert("reasoning_summary".to_string(), json!(reasoning_summary));
  1363. }
  1364. }
  1365. Value::Object(body)
  1366. }
  1367. ProtocolKind::Anthropic => {
  1368. let mut body = serde_json::Map::new();
  1369. body.insert("max_tokens".to_string(), json!(request.max_tokens));
  1370. if request.stream {
  1371. body.insert("stream".to_string(), json!(true));
  1372. }
  1373. if let Some(thinking) = &request.thinking
  1374. && thinking.enabled
  1375. {
  1376. let mut thinking_body = serde_json::Map::new();
  1377. let thinking_type = thinking.kind.as_deref().unwrap_or("enabled");
  1378. thinking_body.insert("type".to_string(), json!(thinking_type));
  1379. if thinking_type == "adaptive" {
  1380. if let Some(effort) = &thinking.effort {
  1381. thinking_body.insert("effort".to_string(), json!(effort));
  1382. }
  1383. } else if let Some(budget_tokens) = thinking.budget_tokens {
  1384. thinking_body.insert("budget_tokens".to_string(), json!(budget_tokens));
  1385. }
  1386. if let Some(display) = &thinking.display {
  1387. thinking_body.insert("display".to_string(), json!(display));
  1388. }
  1389. body.insert("thinking".to_string(), Value::Object(thinking_body));
  1390. } else {
  1391. body.insert("temperature".to_string(), json!(request.temperature));
  1392. }
  1393. Value::Object(body)
  1394. }
  1395. ProtocolKind::Google => {
  1396. let mut generation_config = serde_json::Map::new();
  1397. generation_config.insert("temperature".to_string(), json!(request.temperature));
  1398. generation_config.insert("maxOutputTokens".to_string(), json!(request.max_tokens));
  1399. if let Some(thinking) = &request.thinking
  1400. && thinking.enabled
  1401. {
  1402. let mut thinking_config = serde_json::Map::new();
  1403. if let Some(display) = &thinking.display {
  1404. thinking_config.insert(
  1405. "includeThoughts".to_string(),
  1406. json!(display != "omitted" && display != "false"),
  1407. );
  1408. }
  1409. if let Some(budget_tokens) = thinking.budget_tokens {
  1410. thinking_config.insert("thinkingBudget".to_string(), json!(budget_tokens));
  1411. }
  1412. if let Some(effort) = &thinking.effort {
  1413. thinking_config.insert("thinkingLevel".to_string(), json!(effort));
  1414. }
  1415. if !thinking_config.is_empty() {
  1416. generation_config
  1417. .insert("thinkingConfig".to_string(), Value::Object(thinking_config));
  1418. }
  1419. }
  1420. let mut body = serde_json::Map::new();
  1421. body.insert(
  1422. "generationConfig".to_string(),
  1423. Value::Object(generation_config),
  1424. );
  1425. Value::Object(body)
  1426. }
  1427. }
  1428. }
  1429. fn latency_report(summary: &LatencySummary) -> LatencyReport {
  1430. LatencyReport {
  1431. p50: summary.p50,
  1432. p95: summary.p95,
  1433. p99: summary.p99,
  1434. }
  1435. }
  1436. fn print_benchmark_report(report: &BenchmarkReport, report_path: &Path) {
  1437. println!("benchmark: {}", report.benchmark);
  1438. println!(
  1439. "accuracy: {}",
  1440. report
  1441. .summary
  1442. .accuracy
  1443. .map(|accuracy| format!("{:.2}%", accuracy * 100.0))
  1444. .unwrap_or_else(|| "n/a".to_string())
  1445. );
  1446. println!(
  1447. "success: {}/{} (failed: {})",
  1448. report.summary.success, report.summary.total, report.summary.failed
  1449. );
  1450. println!(
  1451. "latency_ms: p50={} p95={} p99={}",
  1452. format_optional_latency(report.summary.latency_ms.p50),
  1453. format_optional_latency(report.summary.latency_ms.p95),
  1454. format_optional_latency(report.summary.latency_ms.p99)
  1455. );
  1456. if report.summary.ttft_ms.p50.is_some() {
  1457. println!(
  1458. "ttft_ms: p50={} p95={} p99={}",
  1459. format_optional_latency(report.summary.ttft_ms.p50),
  1460. format_optional_latency(report.summary.ttft_ms.p95),
  1461. format_optional_latency(report.summary.ttft_ms.p99)
  1462. );
  1463. }
  1464. println!("errors:");
  1465. if report.errors.is_empty() {
  1466. println!(" none");
  1467. } else {
  1468. for error in &report.errors {
  1469. println!(" {}: {}", error.code, error.count);
  1470. }
  1471. }
  1472. println!("wrong_cases:");
  1473. if report.wrong_cases.is_empty() {
  1474. println!(" none");
  1475. } else {
  1476. for case in &report.wrong_cases {
  1477. println!(
  1478. " {} expected={} actual={}",
  1479. case.id, case.expected, case.actual
  1480. );
  1481. }
  1482. }
  1483. println!("report: {}", report_path.display());
  1484. }
  1485. fn print_rpm_report(report: &RpmReport, report_path: &Path) {
  1486. println!("mode: {}", report.mode);
  1487. println!("target_rpm: {}", report.run.target_rpm);
  1488. println!(
  1489. "actual_rpm: {}",
  1490. report
  1491. .run
  1492. .actual_rpm
  1493. .map(|rpm| format!("{rpm:.2}"))
  1494. .unwrap_or_else(|| "n/a".to_string())
  1495. );
  1496. println!("actual_requests: {}", report.summary.actual_requests);
  1497. println!(
  1498. "success: {} failed: {}",
  1499. report.summary.success, report.summary.failure
  1500. );
  1501. println!(
  1502. "latency_ms: p50={} p95={} p99={}",
  1503. format_optional_latency(report.summary.latency_ms.p50),
  1504. format_optional_latency(report.summary.latency_ms.p95),
  1505. format_optional_latency(report.summary.latency_ms.p99)
  1506. );
  1507. if report.summary.ttft_ms.p50.is_some() {
  1508. println!(
  1509. "ttft_ms: p50={} p95={} p99={}",
  1510. format_optional_latency(report.summary.ttft_ms.p50),
  1511. format_optional_latency(report.summary.ttft_ms.p95),
  1512. format_optional_latency(report.summary.ttft_ms.p99)
  1513. );
  1514. }
  1515. println!("errors:");
  1516. if report.errors.is_empty() {
  1517. println!(" none");
  1518. } else {
  1519. for error in &report.errors {
  1520. println!(" {}: {}", error.code, error.count);
  1521. }
  1522. }
  1523. println!("report: {}", report_path.display());
  1524. }
  1525. fn format_optional_latency(value: Option<u64>) -> String {
  1526. value
  1527. .map(|value| value.to_string())
  1528. .unwrap_or_else(|| "n/a".to_string())
  1529. }
  1530. fn error_code(error: &anyhow::Error) -> String {
  1531. let message = error.to_string();
  1532. let status_regex = Regex::new(r"status\s+(\d{3})").expect("valid status regex");
  1533. status_regex
  1534. .captures(&message)
  1535. .and_then(|captures| captures.get(1))
  1536. .map(|code| code.as_str().to_string())
  1537. .unwrap_or_else(|| "request_error".to_string())
  1538. }
  1539. fn nonzero_concurrency(concurrency: usize) -> usize {
  1540. concurrency.max(1)
  1541. }
  1542. fn parse_duration(value: &str) -> Result<Duration> {
  1543. let value = value.trim();
  1544. let Some(number) = value.strip_suffix('s') else {
  1545. if let Some(number) = value.strip_suffix('m') {
  1546. let minutes = number
  1547. .parse::<u64>()
  1548. .with_context(|| format!("invalid duration: {value}"))?;
  1549. return Ok(Duration::from_secs(minutes * 60));
  1550. }
  1551. bail!("invalid duration: expected values like 60s or 5m");
  1552. };
  1553. let seconds = number
  1554. .parse::<u64>()
  1555. .with_context(|| format!("invalid duration: {value}"))?;
  1556. Ok(Duration::from_secs(seconds))
  1557. }
  1558. fn dataset_data_dir(config_path: &Path) -> Result<PathBuf> {
  1559. if !config_path.exists() {
  1560. return Ok(PathBuf::from("data/benchmarks"));
  1561. }
  1562. let config = AppConfig::load(config_path)?;
  1563. Ok(PathBuf::from(config.benchmarks.data_dir))
  1564. }
  1565. #[cfg(test)]
  1566. mod tests {
  1567. use super::*;
  1568. #[test]
  1569. fn dataset_data_dir_defaults_when_config_is_missing() {
  1570. let temp_dir = tempfile::tempdir().expect("create temp dir");
  1571. let missing_config = temp_dir.path().join("missing-config.yaml");
  1572. let data_dir = dataset_data_dir(&missing_config).expect("default data dir");
  1573. assert_eq!(data_dir, PathBuf::from("data/benchmarks"));
  1574. }
  1575. #[test]
  1576. fn dataset_data_dir_propagates_invalid_existing_config() {
  1577. let temp_dir = tempfile::tempdir().expect("create temp dir");
  1578. let config_path = temp_dir.path().join("config.yaml");
  1579. std::fs::write(&config_path, "providers: [").expect("write invalid config");
  1580. let error = dataset_data_dir(&config_path).expect_err("invalid config should fail");
  1581. assert!(error.to_string().contains("failed to parse config"));
  1582. }
  1583. #[test]
  1584. fn parses_duration_seconds_and_minutes() {
  1585. assert_eq!(parse_duration("60s").expect("seconds").as_secs(), 60);
  1586. assert_eq!(parse_duration("5m").expect("minutes").as_secs(), 300);
  1587. }
  1588. #[test]
  1589. fn computes_rpm_start_schedule_from_run_start() {
  1590. let schedule = sustained_schedule(Duration::from_secs(60), 120);
  1591. assert_eq!(schedule.len(), 120);
  1592. assert_eq!(schedule[0], Duration::ZERO);
  1593. assert_eq!(schedule[1], Duration::from_millis(500));
  1594. assert_eq!(schedule[119], Duration::from_millis(59_500));
  1595. }
  1596. #[test]
  1597. fn rejects_invalid_duration() {
  1598. let error = parse_duration("one hour").expect_err("invalid duration");
  1599. assert!(error.to_string().contains("duration"));
  1600. }
  1601. #[test]
  1602. fn rpm_command_defaults_to_sustained_mode() {
  1603. let cli = Cli::try_parse_from([
  1604. "lq_token_test",
  1605. "rpm",
  1606. "--provider",
  1607. "anthropic",
  1608. "--rpm",
  1609. "120",
  1610. "--prompt",
  1611. "hello",
  1612. ])
  1613. .expect("parse rpm");
  1614. let Command::Rpm { mode, rpm, .. } = cli.command else {
  1615. panic!("expected rpm command");
  1616. };
  1617. assert_eq!(mode, RpmMode::Sustained);
  1618. assert_eq!(rpm, Some(120));
  1619. }
  1620. #[test]
  1621. fn rpm_command_parses_token_bucket_mode() {
  1622. let cli = Cli::try_parse_from([
  1623. "lq_token_test",
  1624. "rpm",
  1625. "--mode",
  1626. "token-bucket",
  1627. "--provider",
  1628. "anthropic",
  1629. "--rpm",
  1630. "120",
  1631. "--burst",
  1632. "120",
  1633. "--probe-seconds",
  1634. "30",
  1635. "--prompt",
  1636. "hello",
  1637. ])
  1638. .expect("parse token bucket rpm");
  1639. let Command::Rpm {
  1640. mode,
  1641. burst,
  1642. probe_seconds,
  1643. ..
  1644. } = cli.command
  1645. else {
  1646. panic!("expected rpm command");
  1647. };
  1648. assert_eq!(mode, RpmMode::TokenBucket);
  1649. assert_eq!(burst, Some(120));
  1650. assert_eq!(probe_seconds, Some(30));
  1651. }
  1652. #[test]
  1653. fn check_command_parses_bare_stream_as_true() {
  1654. let cli = Cli::try_parse_from([
  1655. "lq_token_test",
  1656. "check",
  1657. "--provider",
  1658. "anthropic",
  1659. "--prompt",
  1660. "hello",
  1661. "--stream",
  1662. ])
  1663. .expect("parse check stream");
  1664. let Command::Check { stream, .. } = cli.command else {
  1665. panic!("expected check command");
  1666. };
  1667. assert_eq!(stream, Some(true));
  1668. }
  1669. #[test]
  1670. fn bench_command_parses_explicit_stream_false() {
  1671. let cli = Cli::try_parse_from([
  1672. "lq_token_test",
  1673. "bench",
  1674. "aime2026",
  1675. "--provider",
  1676. "anthropic",
  1677. "--stream",
  1678. "false",
  1679. ])
  1680. .expect("parse bench stream");
  1681. let Command::Bench {
  1682. command: BenchCommand::Aime2026 { stream, .. },
  1683. } = cli.command
  1684. else {
  1685. panic!("expected aime2026 bench command");
  1686. };
  1687. assert_eq!(stream, Some(false));
  1688. }
  1689. #[test]
  1690. fn bench_command_defaults_max_tokens_to_64000() {
  1691. let cli = Cli::try_parse_from(["lq_token_test", "bench", "gpqa-diamond"])
  1692. .expect("parse gpqa bench");
  1693. let Command::Bench {
  1694. command: BenchCommand::GpqaDiamond { max_tokens, .. },
  1695. } = cli.command
  1696. else {
  1697. panic!("expected gpqa-diamond bench command");
  1698. };
  1699. assert_eq!(max_tokens, 64_000);
  1700. }
  1701. #[test]
  1702. fn check_command_parses_debug_raw() {
  1703. let cli = Cli::try_parse_from([
  1704. "lq_token_test",
  1705. "check",
  1706. "--provider",
  1707. "anthropic",
  1708. "--prompt",
  1709. "hello",
  1710. "--debug-raw",
  1711. ])
  1712. .expect("parse check debug raw");
  1713. let Command::Check { debug_raw, .. } = cli.command else {
  1714. panic!("expected check command");
  1715. };
  1716. assert!(debug_raw);
  1717. }
  1718. #[test]
  1719. fn raw_debug_config_is_created_even_when_success_raw_is_disabled() {
  1720. let raw_debug = raw_debug_config(false, "google", "gemini-test");
  1721. assert!(raw_debug.is_some());
  1722. }
  1723. #[test]
  1724. fn check_command_parses_thinking_overrides() {
  1725. let cli = Cli::try_parse_from([
  1726. "lq_token_test",
  1727. "check",
  1728. "--provider",
  1729. "anthropic",
  1730. "--prompt",
  1731. "hello",
  1732. "--thinking",
  1733. "true",
  1734. "--thinking-type",
  1735. "adaptive",
  1736. "--thinking-budget-tokens",
  1737. "10000",
  1738. "--thinking-effort",
  1739. "high",
  1740. "--thinking-display",
  1741. "omitted",
  1742. "--reasoning-effort",
  1743. "high",
  1744. "--reasoning-summary",
  1745. "auto",
  1746. ])
  1747. .expect("parse thinking options");
  1748. let Command::Check {
  1749. thinking,
  1750. thinking_type,
  1751. thinking_budget_tokens,
  1752. thinking_effort,
  1753. thinking_display,
  1754. reasoning_effort,
  1755. reasoning_summary,
  1756. ..
  1757. } = cli.command
  1758. else {
  1759. panic!("expected check command");
  1760. };
  1761. assert_eq!(thinking, Some(true));
  1762. assert_eq!(thinking_type.as_deref(), Some("adaptive"));
  1763. assert_eq!(thinking_budget_tokens, Some(10000));
  1764. assert_eq!(thinking_effort.as_deref(), Some("high"));
  1765. assert_eq!(thinking_display.as_deref(), Some("omitted"));
  1766. assert_eq!(reasoning_effort.as_deref(), Some("high"));
  1767. assert_eq!(reasoning_summary.as_deref(), Some("auto"));
  1768. }
  1769. #[test]
  1770. fn check_command_parses_bare_thinking_as_true() {
  1771. let cli = Cli::try_parse_from([
  1772. "lq_token_test",
  1773. "check",
  1774. "--provider",
  1775. "anthropic",
  1776. "--prompt",
  1777. "hello",
  1778. "--thinking",
  1779. ])
  1780. .expect("parse bare thinking option");
  1781. let Command::Check { thinking, .. } = cli.command else {
  1782. panic!("expected check command");
  1783. };
  1784. assert_eq!(thinking, Some(true));
  1785. }
  1786. #[test]
  1787. fn disabled_thinking_report_keeps_cli_budget_override() {
  1788. let provider = ProviderThinkingConfig {
  1789. enabled: Some(true),
  1790. kind: Some("enabled".to_string()),
  1791. budget_tokens: Some(10000),
  1792. effort: None,
  1793. display: None,
  1794. reasoning_effort: None,
  1795. reasoning_summary: None,
  1796. };
  1797. let merged = merged_thinking_config(
  1798. Some(&provider),
  1799. ThinkingOverrides {
  1800. thinking: Some(false),
  1801. thinking_budget_tokens: Some(20000),
  1802. ..ThinkingOverrides::default()
  1803. },
  1804. )
  1805. .expect("disabled thinking config");
  1806. assert!(!merged.enabled);
  1807. assert_eq!(merged.budget_tokens, Some(20000));
  1808. }
  1809. #[test]
  1810. fn sent_anthropic_adaptive_request_params_omit_temperature_and_nulls() {
  1811. let request = ModelRequest {
  1812. base_url: "https://example.test".to_string(),
  1813. api_token: "secret".to_string(),
  1814. model: "claude-test".to_string(),
  1815. prompt: "hello".to_string(),
  1816. temperature: 0.0,
  1817. max_tokens: 32_768,
  1818. stream: true,
  1819. raw_debug: None,
  1820. thinking: Some(ThinkingConfig {
  1821. enabled: true,
  1822. kind: Some("adaptive".to_string()),
  1823. budget_tokens: None,
  1824. effort: Some("high".to_string()),
  1825. display: Some("summarized".to_string()),
  1826. reasoning_effort: None,
  1827. reasoning_summary: None,
  1828. }),
  1829. };
  1830. let params = sent_request_params(ProtocolKind::Anthropic, &request);
  1831. assert_eq!(params.protocol, "anthropic");
  1832. assert_eq!(params.body["max_tokens"], 32_768);
  1833. assert_eq!(params.body["stream"], true);
  1834. assert_eq!(params.body["thinking"]["type"], "adaptive");
  1835. assert_eq!(params.body["thinking"]["effort"], "high");
  1836. assert_eq!(params.body["thinking"]["display"], "summarized");
  1837. assert!(params.body.get("temperature").is_none());
  1838. assert!(params.body["thinking"].get("budget_tokens").is_none());
  1839. assert!(params.body["thinking"].get("reasoning_effort").is_none());
  1840. }
  1841. #[test]
  1842. fn sent_google_request_params_use_generation_config_names() {
  1843. let request = ModelRequest {
  1844. base_url: "https://example.test".to_string(),
  1845. api_token: "secret".to_string(),
  1846. model: "gemini-test".to_string(),
  1847. prompt: "hello".to_string(),
  1848. temperature: 0.0,
  1849. max_tokens: 32_768,
  1850. stream: true,
  1851. raw_debug: None,
  1852. thinking: Some(ThinkingConfig {
  1853. enabled: true,
  1854. kind: None,
  1855. budget_tokens: Some(5000),
  1856. effort: Some("high".to_string()),
  1857. display: Some("summarized".to_string()),
  1858. reasoning_effort: None,
  1859. reasoning_summary: None,
  1860. }),
  1861. };
  1862. let params = sent_request_params(ProtocolKind::Google, &request);
  1863. assert_eq!(params.protocol, "google");
  1864. assert_eq!(params.body["generationConfig"]["temperature"], 0.0);
  1865. assert_eq!(params.body["generationConfig"]["maxOutputTokens"], 32_768);
  1866. assert_eq!(
  1867. params.body["generationConfig"]["thinkingConfig"]["thinkingBudget"],
  1868. 5000
  1869. );
  1870. assert_eq!(
  1871. params.body["generationConfig"]["thinkingConfig"]["thinkingLevel"],
  1872. "high"
  1873. );
  1874. assert_eq!(
  1875. params.body["generationConfig"]["thinkingConfig"]["includeThoughts"],
  1876. true
  1877. );
  1878. }
  1879. #[test]
  1880. fn response_debug_report_records_near_ttft_latency_with_full_prompt_and_output() {
  1881. let response = ModelResponse {
  1882. text: "final answer".to_string(),
  1883. status: 200,
  1884. elapsed_ms: 1_500,
  1885. first_token_ms: Some(750),
  1886. };
  1887. let debug = response_debug_report(
  1888. Some("case-1".to_string()),
  1889. None,
  1890. None,
  1891. "full prompt".to_string(),
  1892. &response,
  1893. )
  1894. .expect("near ttft/latency should be debugged");
  1895. assert_eq!(debug.reason, "near_ttft_latency");
  1896. assert_eq!(debug.id.as_deref(), Some("case-1"));
  1897. assert_eq!(debug.prompt, "full prompt");
  1898. assert_eq!(debug.output, "final answer");
  1899. assert_eq!(debug.latency_ms, 1_500);
  1900. assert_eq!(debug.ttft_ms, Some(750));
  1901. assert_eq!(debug.ttft_latency_delta_ms, Some(750));
  1902. assert!(!debug.has_think_tags);
  1903. }
  1904. #[test]
  1905. fn response_debug_report_marks_think_tags() {
  1906. let response = ModelResponse {
  1907. text: "<think>hidden</think>\nanswer".to_string(),
  1908. status: 200,
  1909. elapsed_ms: 5_000,
  1910. first_token_ms: Some(100),
  1911. };
  1912. let debug = response_debug_report(None, None, None, "prompt".to_string(), &response)
  1913. .expect("think tags should be debugged");
  1914. assert_eq!(debug.reason, "contains_think_tags");
  1915. assert!(debug.has_think_tags);
  1916. assert_eq!(debug.ttft_latency_delta_ms, Some(4_900));
  1917. }
  1918. #[test]
  1919. fn response_debug_report_records_empty_content_as_request_debug() {
  1920. let response = ModelResponse {
  1921. text: " ".to_string(),
  1922. status: 200,
  1923. elapsed_ms: 2_000,
  1924. first_token_ms: None,
  1925. };
  1926. let debug = response_debug_report(
  1927. None,
  1928. Some("burst".to_string()),
  1929. Some(0),
  1930. "prompt".to_string(),
  1931. &response,
  1932. )
  1933. .expect("empty content should be debugged");
  1934. assert_eq!(debug.reason, "no_content");
  1935. assert_eq!(debug.phase.as_deref(), Some("burst"));
  1936. assert_eq!(debug.second, Some(0));
  1937. assert!(response_has_no_content(&response));
  1938. }
  1939. #[test]
  1940. fn rpm_command_parses_window_boundary_offset() {
  1941. let cli = Cli::try_parse_from([
  1942. "lq_token_test",
  1943. "rpm",
  1944. "--mode",
  1945. "window-boundary",
  1946. "--provider",
  1947. "anthropic",
  1948. "--burst",
  1949. "10",
  1950. "--window-offset-ms",
  1951. "250",
  1952. "--prompt",
  1953. "hello",
  1954. ])
  1955. .expect("parse window boundary rpm");
  1956. let Command::Rpm {
  1957. mode,
  1958. window_offset_ms,
  1959. ..
  1960. } = cli.command
  1961. else {
  1962. panic!("expected rpm command");
  1963. };
  1964. assert_eq!(mode, RpmMode::WindowBoundary);
  1965. assert_eq!(window_offset_ms, 250);
  1966. }
  1967. #[test]
  1968. fn rpm_mode_validation_rejects_zero_values() {
  1969. let zero_rpm = build_rpm_mode_plan(RpmMode::Sustained, Some(0), "60s", None, None, 500)
  1970. .expect_err("zero rpm should fail");
  1971. let zero_burst = build_rpm_mode_plan(RpmMode::Burst, None, "60s", Some(0), None, 500)
  1972. .expect_err("zero burst should fail");
  1973. assert!(zero_rpm.to_string().contains("rpm"));
  1974. assert!(zero_burst.to_string().contains("burst"));
  1975. }
  1976. }