api模型检测
No puede seleccionar más de 25 temas Los temas deben comenzar con una letra o número, pueden incluir guiones ('-') y pueden tener hasta 35 caracteres de largo.
 
 

666 líneas
21 KiB

  1. use crate::benchmarks;
  2. use crate::benchmarks::judge;
  3. use crate::config::AppConfig;
  4. use crate::metrics::{LatencySummary, Metrics, MetricsSummary};
  5. use crate::report::{
  6. BenchmarkReport, BenchmarkSummaryReport, DatasetReport, LatencyReport, RpmReport, RpmRunReport,
  7. RpmSummaryReport, RunReport, WrongCaseReport, write_benchmark_report, write_rpm_report,
  8. };
  9. use crate::runner::{ModelRequest, run_model_request};
  10. use anyhow::{Context, Result, bail};
  11. use chrono::Utc;
  12. use clap::{Parser, Subcommand};
  13. use futures::{StreamExt, stream};
  14. use regex::Regex;
  15. use std::path::{Path, PathBuf};
  16. use std::time::{Duration, Instant};
  17. use tokio::time::{Instant as TokioInstant, sleep_until};
  18. #[derive(Debug, Parser)]
  19. #[command(
  20. name = "lq_token_test",
  21. version,
  22. about = "Test LLM relay protocols, RPM, and benchmark accuracy"
  23. )]
  24. pub struct Cli {
  25. #[command(subcommand)]
  26. pub command: Command,
  27. }
  28. #[derive(Debug, Subcommand)]
  29. pub enum Command {
  30. Check {
  31. #[arg(long, default_value = "config.yaml")]
  32. config: PathBuf,
  33. #[arg(long)]
  34. provider: Option<String>,
  35. #[arg(long)]
  36. model: Option<String>,
  37. #[arg(long)]
  38. prompt: String,
  39. },
  40. Dataset {
  41. #[command(subcommand)]
  42. command: DatasetCommand,
  43. },
  44. Bench {
  45. #[command(subcommand)]
  46. command: BenchCommand,
  47. },
  48. Rpm {
  49. #[arg(long, default_value = "config.yaml")]
  50. config: PathBuf,
  51. #[arg(long)]
  52. provider: Option<String>,
  53. #[arg(long)]
  54. model: Option<String>,
  55. #[arg(long)]
  56. rpm: u32,
  57. #[arg(long)]
  58. duration: String,
  59. #[arg(long)]
  60. prompt: String,
  61. },
  62. }
  63. #[derive(Debug, Subcommand)]
  64. pub enum DatasetCommand {
  65. Fetch { dataset: String },
  66. }
  67. #[derive(Debug, Subcommand)]
  68. pub enum BenchCommand {
  69. Aime2026 {
  70. #[arg(long, default_value = "config.yaml")]
  71. config: PathBuf,
  72. #[arg(long)]
  73. provider: Option<String>,
  74. #[arg(long)]
  75. model: Option<String>,
  76. #[arg(long, default_value_t = 4)]
  77. concurrency: usize,
  78. #[arg(long)]
  79. limit: Option<usize>,
  80. },
  81. GpqaDiamond {
  82. #[arg(long, default_value = "config.yaml")]
  83. config: PathBuf,
  84. #[arg(long)]
  85. provider: Option<String>,
  86. #[arg(long)]
  87. model: Option<String>,
  88. #[arg(long, default_value_t = 4)]
  89. concurrency: usize,
  90. #[arg(long)]
  91. limit: Option<usize>,
  92. },
  93. }
  94. pub async fn dispatch(cli: Cli) -> Result<()> {
  95. match cli.command {
  96. Command::Check {
  97. config,
  98. provider,
  99. model,
  100. prompt,
  101. } => {
  102. let config = AppConfig::load(&config)?;
  103. let provider = config.resolved_provider(provider.as_deref())?;
  104. let request = ModelRequest {
  105. base_url: provider.base_url.clone(),
  106. api_token: provider.api_token.clone(),
  107. model: model.unwrap_or_else(|| provider.default_model.clone()),
  108. prompt,
  109. temperature: 0.0,
  110. max_tokens: 1024,
  111. };
  112. let response = run_model_request(provider.protocol, request).await?;
  113. println!("status: {}", response.status);
  114. println!("elapsed_ms: {}", response.elapsed_ms);
  115. println!("{}", response.text);
  116. Ok(())
  117. }
  118. Command::Dataset {
  119. command: DatasetCommand::Fetch { dataset },
  120. } => {
  121. let data_dir = dataset_data_dir(Path::new("config.yaml"))?;
  122. let path = benchmarks::fetch_dataset(&dataset, &data_dir).await?;
  123. println!("{}", path.display());
  124. Ok(())
  125. }
  126. Command::Bench { command } => dispatch_bench(command).await,
  127. Command::Rpm {
  128. config,
  129. provider,
  130. model,
  131. rpm,
  132. duration,
  133. prompt,
  134. } => run_rpm(config, provider, model, rpm, duration, prompt).await,
  135. }
  136. }
  137. async fn dispatch_bench(command: BenchCommand) -> Result<()> {
  138. match command {
  139. BenchCommand::Aime2026 {
  140. config,
  141. provider,
  142. model,
  143. concurrency,
  144. limit,
  145. } => run_aime_benchmark(config, provider, model, concurrency, limit).await,
  146. BenchCommand::GpqaDiamond {
  147. config,
  148. provider,
  149. model,
  150. concurrency,
  151. limit,
  152. } => run_gpqa_benchmark(config, provider, model, concurrency, limit).await,
  153. }
  154. }
  155. async fn run_aime_benchmark(
  156. config_path: PathBuf,
  157. provider: Option<String>,
  158. model: Option<String>,
  159. concurrency: usize,
  160. limit: Option<usize>,
  161. ) -> Result<()> {
  162. let config = AppConfig::load(&config_path)?;
  163. let provider_name = provider_name(&config, provider.as_deref())?;
  164. let provider_config = config.resolved_provider(Some(&provider_name))?;
  165. let model = model.unwrap_or_else(|| provider_config.default_model.clone());
  166. let loaded = benchmarks::aime::load_cases(Path::new(&config.benchmarks.data_dir))?;
  167. let dataset = dataset_report(
  168. config
  169. .benchmarks
  170. .aime2026
  171. .as_ref()
  172. .map(|dataset| (dataset.source.as_str(), dataset.split.as_str())),
  173. &loaded.local_path,
  174. );
  175. let cases = apply_limit(loaded.cases, limit);
  176. let started_at = Utc::now();
  177. let started = Instant::now();
  178. let base_request = request_template(&provider_config, &model, 0.0, 1024);
  179. let protocol = provider_config.protocol;
  180. let results = stream::iter(cases)
  181. .map(|case| {
  182. let mut request = base_request.clone();
  183. request.prompt = case.prompt();
  184. async move {
  185. let result = run_model_request(protocol, request).await;
  186. (case, result)
  187. }
  188. })
  189. .buffer_unordered(nonzero_concurrency(concurrency))
  190. .collect::<Vec<_>>()
  191. .await;
  192. let mut metrics = Metrics::new();
  193. let mut wrong_cases = Vec::new();
  194. for (case, result) in results {
  195. match result {
  196. Ok(response) => {
  197. metrics.record_success(response.status, response.elapsed_ms as u64);
  198. let actual = judge::extract_final_integer(&response.text)
  199. .unwrap_or_else(|| "no_answer".to_string());
  200. let correct = judge::judge_integer(&response.text, &case.answer);
  201. metrics.record_judgement(correct);
  202. if !correct {
  203. wrong_cases.push(WrongCaseReport {
  204. id: case.id,
  205. question: case.problem,
  206. expected: case.answer,
  207. actual,
  208. raw_output: response.text,
  209. });
  210. }
  211. }
  212. Err(error) => metrics.record_failure(error_code(&error)),
  213. }
  214. }
  215. let summary = metrics.summary();
  216. let report = benchmark_report(BenchmarkReportInput {
  217. benchmark: "aime2026",
  218. provider: provider_name,
  219. model,
  220. dataset,
  221. started_at,
  222. duration_ms: started.elapsed().as_millis(),
  223. concurrency,
  224. limit,
  225. summary,
  226. wrong_cases,
  227. });
  228. let report_path = write_benchmark_report(Path::new("."), &report)?;
  229. print_benchmark_report(&report, &report_path);
  230. Ok(())
  231. }
  232. async fn run_gpqa_benchmark(
  233. config_path: PathBuf,
  234. provider: Option<String>,
  235. model: Option<String>,
  236. concurrency: usize,
  237. limit: Option<usize>,
  238. ) -> Result<()> {
  239. let config = AppConfig::load(&config_path)?;
  240. let provider_name = provider_name(&config, provider.as_deref())?;
  241. let provider_config = config.resolved_provider(Some(&provider_name))?;
  242. let model = model.unwrap_or_else(|| provider_config.default_model.clone());
  243. let loaded = benchmarks::gpqa::load_cases(Path::new(&config.benchmarks.data_dir))?;
  244. let dataset = dataset_report(
  245. config
  246. .benchmarks
  247. .gpqa_diamond
  248. .as_ref()
  249. .map(|dataset| (dataset.source.as_str(), dataset.split.as_str())),
  250. &loaded.local_path,
  251. );
  252. let cases = apply_limit(loaded.cases, limit);
  253. let started_at = Utc::now();
  254. let started = Instant::now();
  255. let base_request = request_template(&provider_config, &model, 0.0, 1024);
  256. let protocol = provider_config.protocol;
  257. let results = stream::iter(cases)
  258. .map(|case| {
  259. let mut request = base_request.clone();
  260. request.prompt = case.prompt();
  261. async move {
  262. let result = run_model_request(protocol, request).await;
  263. (case, result)
  264. }
  265. })
  266. .buffer_unordered(nonzero_concurrency(concurrency))
  267. .collect::<Vec<_>>()
  268. .await;
  269. let mut metrics = Metrics::new();
  270. let mut wrong_cases = Vec::new();
  271. for (case, result) in results {
  272. match result {
  273. Ok(response) => {
  274. metrics.record_success(response.status, response.elapsed_ms as u64);
  275. let actual = judge::extract_choice(&response.text)
  276. .map(|choice| choice.to_string())
  277. .unwrap_or_else(|| "no_answer".to_string());
  278. let expected = case.answer.to_string();
  279. let correct = judge::judge_choice(&response.text, case.answer);
  280. metrics.record_judgement(correct);
  281. if !correct {
  282. wrong_cases.push(WrongCaseReport {
  283. id: case.id,
  284. question: case.question,
  285. expected,
  286. actual,
  287. raw_output: response.text,
  288. });
  289. }
  290. }
  291. Err(error) => metrics.record_failure(error_code(&error)),
  292. }
  293. }
  294. let summary = metrics.summary();
  295. let report = benchmark_report(BenchmarkReportInput {
  296. benchmark: "gpqa-diamond",
  297. provider: provider_name,
  298. model,
  299. dataset,
  300. started_at,
  301. duration_ms: started.elapsed().as_millis(),
  302. concurrency,
  303. limit,
  304. summary,
  305. wrong_cases,
  306. });
  307. let report_path = write_benchmark_report(Path::new("."), &report)?;
  308. print_benchmark_report(&report, &report_path);
  309. Ok(())
  310. }
  311. async fn run_rpm(
  312. config_path: PathBuf,
  313. provider: Option<String>,
  314. model: Option<String>,
  315. rpm: u32,
  316. duration: String,
  317. prompt: String,
  318. ) -> Result<()> {
  319. if rpm == 0 {
  320. bail!("rpm must be greater than 0");
  321. }
  322. let duration = parse_duration(&duration)?;
  323. let config = AppConfig::load(&config_path)?;
  324. let provider_name = provider_name(&config, provider.as_deref())?;
  325. let provider_config = config.resolved_provider(Some(&provider_name))?;
  326. let model = model.unwrap_or_else(|| provider_config.default_model.clone());
  327. let request = ModelRequest {
  328. prompt,
  329. ..request_template(&provider_config, &model, 0.0, 1024)
  330. };
  331. let schedule = rpm_start_schedule(duration, rpm);
  332. let started_at = Utc::now();
  333. let started = Instant::now();
  334. let tokio_started = TokioInstant::now();
  335. let in_flight_limit = schedule.len().max(1);
  336. let mut metrics = Metrics::new();
  337. let results = stream::iter(schedule.into_iter().map(|offset| {
  338. let request = request.clone();
  339. async move {
  340. sleep_until(tokio_started + offset).await;
  341. run_model_request(provider_config.protocol, request).await
  342. }
  343. }))
  344. .buffer_unordered(in_flight_limit)
  345. .collect::<Vec<_>>()
  346. .await;
  347. for result in results {
  348. match result {
  349. Ok(response) => metrics.record_success(response.status, response.elapsed_ms as u64),
  350. Err(error) => metrics.record_failure(error_code(&error)),
  351. }
  352. }
  353. let summary = metrics.summary();
  354. let report = RpmReport {
  355. benchmark: "rpm".to_string(),
  356. provider: provider_name,
  357. model,
  358. run: RpmRunReport {
  359. started_at,
  360. duration_ms: started.elapsed().as_millis(),
  361. target_rpm: rpm,
  362. temperature: 0.0,
  363. max_tokens: 1024,
  364. },
  365. summary: RpmSummaryReport {
  366. actual_requests: summary.total,
  367. success: summary.success,
  368. failure: summary.failed,
  369. latency_ms: latency_report(&summary.latency_ms),
  370. },
  371. errors: summary.errors,
  372. };
  373. let report_path = write_rpm_report(Path::new("."), &report)?;
  374. print_rpm_report(&report, &report_path);
  375. Ok(())
  376. }
  377. fn provider_name(config: &AppConfig, provider: Option<&str>) -> Result<String> {
  378. match provider {
  379. Some(provider) => Ok(provider.to_string()),
  380. None => config
  381. .default_provider
  382. .clone()
  383. .context("no provider specified and config has no default_provider"),
  384. }
  385. }
  386. fn request_template(
  387. provider: &crate::config::ProviderConfig,
  388. model: &str,
  389. temperature: f32,
  390. max_tokens: u32,
  391. ) -> ModelRequest {
  392. ModelRequest {
  393. base_url: provider.base_url.clone(),
  394. api_token: provider.api_token.clone(),
  395. model: model.to_string(),
  396. prompt: String::new(),
  397. temperature,
  398. max_tokens,
  399. }
  400. }
  401. fn apply_limit<T>(cases: Vec<T>, limit: Option<usize>) -> Vec<T> {
  402. match limit {
  403. Some(limit) => cases.into_iter().take(limit).collect(),
  404. None => cases,
  405. }
  406. }
  407. fn dataset_report(config: Option<(&str, &str)>, local_path: &Path) -> DatasetReport {
  408. let (source, split) = config.unwrap_or(("local", "train"));
  409. DatasetReport {
  410. source: source.to_string(),
  411. split: split.to_string(),
  412. revision: None,
  413. local_path: local_path.display().to_string(),
  414. }
  415. }
  416. struct BenchmarkReportInput {
  417. benchmark: &'static str,
  418. provider: String,
  419. model: String,
  420. dataset: DatasetReport,
  421. started_at: chrono::DateTime<Utc>,
  422. duration_ms: u128,
  423. concurrency: usize,
  424. limit: Option<usize>,
  425. summary: MetricsSummary,
  426. wrong_cases: Vec<WrongCaseReport>,
  427. }
  428. fn benchmark_report(input: BenchmarkReportInput) -> BenchmarkReport {
  429. BenchmarkReport {
  430. benchmark: input.benchmark.to_string(),
  431. provider: input.provider,
  432. model: input.model,
  433. dataset: input.dataset,
  434. run: RunReport {
  435. started_at: input.started_at,
  436. duration_ms: input.duration_ms,
  437. concurrency: input.concurrency,
  438. limit: input.limit,
  439. temperature: 0.0,
  440. max_tokens: 1024,
  441. },
  442. summary: BenchmarkSummaryReport {
  443. accuracy: input.summary.accuracy,
  444. success: input.summary.success,
  445. total: input.summary.total,
  446. correct: input.summary.correct,
  447. wrong: input.summary.wrong,
  448. failed: input.summary.failed,
  449. latency_ms: latency_report(&input.summary.latency_ms),
  450. },
  451. errors: input.summary.errors,
  452. wrong_cases: input.wrong_cases,
  453. }
  454. }
  455. fn latency_report(summary: &LatencySummary) -> LatencyReport {
  456. LatencyReport {
  457. p50: summary.p50,
  458. p95: summary.p95,
  459. p99: summary.p99,
  460. }
  461. }
  462. fn print_benchmark_report(report: &BenchmarkReport, report_path: &Path) {
  463. println!("benchmark: {}", report.benchmark);
  464. println!(
  465. "accuracy: {}",
  466. report
  467. .summary
  468. .accuracy
  469. .map(|accuracy| format!("{:.2}%", accuracy * 100.0))
  470. .unwrap_or_else(|| "n/a".to_string())
  471. );
  472. println!(
  473. "success: {}/{} (failed: {})",
  474. report.summary.success, report.summary.total, report.summary.failed
  475. );
  476. println!(
  477. "latency_ms: p50={} p95={} p99={}",
  478. format_optional_latency(report.summary.latency_ms.p50),
  479. format_optional_latency(report.summary.latency_ms.p95),
  480. format_optional_latency(report.summary.latency_ms.p99)
  481. );
  482. println!("errors:");
  483. if report.errors.is_empty() {
  484. println!(" none");
  485. } else {
  486. for error in &report.errors {
  487. println!(" {}: {}", error.code, error.count);
  488. }
  489. }
  490. println!("wrong_cases:");
  491. if report.wrong_cases.is_empty() {
  492. println!(" none");
  493. } else {
  494. for case in &report.wrong_cases {
  495. println!(
  496. " {} expected={} actual={}",
  497. case.id, case.expected, case.actual
  498. );
  499. }
  500. }
  501. println!("report: {}", report_path.display());
  502. }
  503. fn print_rpm_report(report: &RpmReport, report_path: &Path) {
  504. println!("target_rpm: {}", report.run.target_rpm);
  505. println!("actual_requests: {}", report.summary.actual_requests);
  506. println!(
  507. "success: {} failed: {}",
  508. report.summary.success, report.summary.failure
  509. );
  510. println!(
  511. "latency_ms: p50={} p95={} p99={}",
  512. format_optional_latency(report.summary.latency_ms.p50),
  513. format_optional_latency(report.summary.latency_ms.p95),
  514. format_optional_latency(report.summary.latency_ms.p99)
  515. );
  516. println!("errors:");
  517. if report.errors.is_empty() {
  518. println!(" none");
  519. } else {
  520. for error in &report.errors {
  521. println!(" {}: {}", error.code, error.count);
  522. }
  523. }
  524. println!("report: {}", report_path.display());
  525. }
  526. fn format_optional_latency(value: Option<u64>) -> String {
  527. value
  528. .map(|value| value.to_string())
  529. .unwrap_or_else(|| "n/a".to_string())
  530. }
  531. fn error_code(error: &anyhow::Error) -> String {
  532. let message = error.to_string();
  533. let status_regex = Regex::new(r"status\s+(\d{3})").expect("valid status regex");
  534. status_regex
  535. .captures(&message)
  536. .and_then(|captures| captures.get(1))
  537. .map(|code| code.as_str().to_string())
  538. .unwrap_or_else(|| "request_error".to_string())
  539. }
  540. fn nonzero_concurrency(concurrency: usize) -> usize {
  541. concurrency.max(1)
  542. }
  543. fn parse_duration(value: &str) -> Result<Duration> {
  544. let value = value.trim();
  545. let Some(number) = value.strip_suffix('s') else {
  546. if let Some(number) = value.strip_suffix('m') {
  547. let minutes = number
  548. .parse::<u64>()
  549. .with_context(|| format!("invalid duration: {value}"))?;
  550. return Ok(Duration::from_secs(minutes * 60));
  551. }
  552. bail!("invalid duration: expected values like 60s or 5m");
  553. };
  554. let seconds = number
  555. .parse::<u64>()
  556. .with_context(|| format!("invalid duration: {value}"))?;
  557. Ok(Duration::from_secs(seconds))
  558. }
  559. fn rpm_start_schedule(duration: Duration, rpm: u32) -> Vec<Duration> {
  560. let interval_nanos = 60_000_000_000u128 / u128::from(rpm);
  561. if interval_nanos == 0 {
  562. return Vec::new();
  563. }
  564. let duration_nanos = duration.as_nanos();
  565. let mut offsets = Vec::new();
  566. let mut offset = 0u128;
  567. while offset < duration_nanos {
  568. offsets.push(Duration::from_nanos(offset.min(u128::from(u64::MAX)) as u64));
  569. offset += interval_nanos;
  570. }
  571. offsets
  572. }
  573. fn dataset_data_dir(config_path: &Path) -> Result<PathBuf> {
  574. if !config_path.exists() {
  575. return Ok(PathBuf::from("data/benchmarks"));
  576. }
  577. let config = AppConfig::load(config_path)?;
  578. Ok(PathBuf::from(config.benchmarks.data_dir))
  579. }
  580. #[cfg(test)]
  581. mod tests {
  582. use super::*;
  583. #[test]
  584. fn dataset_data_dir_defaults_when_config_is_missing() {
  585. let temp_dir = tempfile::tempdir().expect("create temp dir");
  586. let missing_config = temp_dir.path().join("missing-config.yaml");
  587. let data_dir = dataset_data_dir(&missing_config).expect("default data dir");
  588. assert_eq!(data_dir, PathBuf::from("data/benchmarks"));
  589. }
  590. #[test]
  591. fn dataset_data_dir_propagates_invalid_existing_config() {
  592. let temp_dir = tempfile::tempdir().expect("create temp dir");
  593. let config_path = temp_dir.path().join("config.yaml");
  594. std::fs::write(&config_path, "providers: [").expect("write invalid config");
  595. let error = dataset_data_dir(&config_path).expect_err("invalid config should fail");
  596. assert!(error.to_string().contains("failed to parse config"));
  597. }
  598. #[test]
  599. fn parses_duration_seconds_and_minutes() {
  600. assert_eq!(parse_duration("60s").expect("seconds").as_secs(), 60);
  601. assert_eq!(parse_duration("5m").expect("minutes").as_secs(), 300);
  602. }
  603. #[test]
  604. fn computes_rpm_start_schedule_from_run_start() {
  605. let schedule = rpm_start_schedule(Duration::from_secs(60), 120);
  606. assert_eq!(schedule.len(), 120);
  607. assert_eq!(schedule[0], Duration::ZERO);
  608. assert_eq!(schedule[1], Duration::from_millis(500));
  609. assert_eq!(schedule[119], Duration::from_millis(59_500));
  610. }
  611. #[test]
  612. fn rejects_invalid_duration() {
  613. let error = parse_duration("one hour").expect_err("invalid duration");
  614. assert!(error.to_string().contains("duration"));
  615. }
  616. }