api模型检测
Nie możesz wybrać więcej, niż 25 tematów Tematy muszą się zaczynać od litery lub cyfry, mogą zawierać myślniki ('-') i mogą mieć do 35 znaków.

455 wiersze
13 KiB

  1. use crate::metrics::ErrorCount;
  2. use anyhow::{Context, Result};
  3. use chrono::{DateTime, Utc};
  4. use serde::Serialize;
  5. use std::path::{Path, PathBuf};
  6. #[derive(Debug, Clone, Serialize)]
  7. pub struct DatasetReport {
  8. pub source: String,
  9. pub split: String,
  10. pub revision: Option<String>,
  11. pub local_path: String,
  12. }
  13. #[derive(Debug, Clone, Serialize)]
  14. pub struct RunReport {
  15. pub started_at: DateTime<Utc>,
  16. pub duration_ms: u128,
  17. pub concurrency: usize,
  18. pub limit: Option<usize>,
  19. pub temperature: f32,
  20. pub max_tokens: u32,
  21. }
  22. #[derive(Debug, Clone, Serialize)]
  23. pub struct LatencyReport {
  24. pub p50: Option<u64>,
  25. pub p95: Option<u64>,
  26. pub p99: Option<u64>,
  27. }
  28. #[derive(Debug, Clone, Serialize)]
  29. pub struct BenchmarkSummaryReport {
  30. pub accuracy: Option<f64>,
  31. pub success: u64,
  32. pub total: u64,
  33. pub correct: u64,
  34. pub wrong: u64,
  35. pub failed: u64,
  36. pub latency_ms: LatencyReport,
  37. pub ttft_ms: LatencyReport,
  38. }
  39. #[derive(Debug, Clone, Serialize)]
  40. pub struct WrongCaseReport {
  41. pub id: String,
  42. pub question: String,
  43. pub expected: String,
  44. pub actual: String,
  45. pub raw_output: String,
  46. }
  47. #[derive(Debug, Clone, Serialize)]
  48. pub struct CorrectCaseReport {
  49. pub id: String,
  50. pub question: String,
  51. pub expected: String,
  52. pub raw_output: String,
  53. }
  54. #[derive(Debug, Clone, Serialize)]
  55. pub struct BenchmarkReport {
  56. pub benchmark: String,
  57. pub provider: String,
  58. pub model: String,
  59. pub params: BenchmarkParamsReport,
  60. pub dataset: DatasetReport,
  61. pub run: RunReport,
  62. pub summary: BenchmarkSummaryReport,
  63. pub errors: Vec<ErrorCount>,
  64. pub correct_samples: Vec<CorrectCaseReport>,
  65. pub wrong_cases: Vec<WrongCaseReport>,
  66. }
  67. #[derive(Debug, Clone, Serialize)]
  68. pub struct BenchmarkParamsReport {
  69. pub stream: bool,
  70. pub thinking: Option<ThinkingParamsReport>,
  71. }
  72. #[derive(Debug, Clone, Serialize)]
  73. pub struct RpmParamsReport {
  74. pub prompt: String,
  75. pub stream: bool,
  76. pub thinking: Option<ThinkingParamsReport>,
  77. pub duration: String,
  78. pub burst: Option<u32>,
  79. pub concurrency: usize,
  80. pub probe_seconds: Option<u64>,
  81. pub window_offset_ms: u64,
  82. }
  83. #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
  84. pub struct ThinkingParamsReport {
  85. pub enabled: bool,
  86. #[serde(rename = "type")]
  87. pub kind: Option<String>,
  88. pub budget_tokens: Option<u32>,
  89. pub effort: Option<String>,
  90. pub display: Option<String>,
  91. pub reasoning_effort: Option<String>,
  92. pub reasoning_summary: Option<String>,
  93. }
  94. #[derive(Debug, Clone, Serialize)]
  95. pub struct RpmRunReport {
  96. pub started_at: DateTime<Utc>,
  97. pub duration_ms: u128,
  98. pub target_rpm: u32,
  99. pub actual_rpm: Option<f64>,
  100. pub temperature: f32,
  101. pub max_tokens: u32,
  102. }
  103. #[derive(Debug, Clone, Serialize)]
  104. pub struct RpmSummaryReport {
  105. pub actual_requests: u64,
  106. pub success: u64,
  107. pub failure: u64,
  108. pub latency_ms: LatencyReport,
  109. pub ttft_ms: LatencyReport,
  110. }
  111. #[derive(Debug, Clone, Serialize)]
  112. pub struct RpmReport {
  113. pub benchmark: String,
  114. pub mode: String,
  115. pub provider: String,
  116. pub model: String,
  117. pub params: RpmParamsReport,
  118. pub run: RpmRunReport,
  119. pub summary: RpmSummaryReport,
  120. pub mode_detail: Option<RpmModeDetailReport>,
  121. pub errors: Vec<ErrorCount>,
  122. }
  123. #[derive(Debug, Clone, Serialize)]
  124. pub struct RpmModeDetailReport {
  125. pub burst: Option<PhaseSummaryReport>,
  126. pub refill_probe: Vec<ProbeSecondReport>,
  127. pub sliding_probe: Vec<ProbeSecondReport>,
  128. pub window_boundary: Option<WindowBoundaryReport>,
  129. pub inference: Option<LimiterInferenceReport>,
  130. }
  131. #[derive(Debug, Clone, Serialize)]
  132. pub struct PhaseSummaryReport {
  133. pub sent: u64,
  134. pub success: u64,
  135. pub failure: u64,
  136. }
  137. #[derive(Debug, Clone, Serialize)]
  138. pub struct ProbeSecondReport {
  139. pub second: u64,
  140. pub sent: u64,
  141. pub success: u64,
  142. pub failure: u64,
  143. }
  144. #[derive(Debug, Clone, Serialize)]
  145. pub struct WindowBoundaryReport {
  146. pub before: PhaseSummaryReport,
  147. pub after: PhaseSummaryReport,
  148. }
  149. #[derive(Debug, Clone, Serialize)]
  150. pub struct LimiterInferenceReport {
  151. pub likely_limiter: LimiterInferenceKind,
  152. pub confidence: String,
  153. pub signals: Vec<String>,
  154. }
  155. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
  156. #[serde(rename_all = "snake_case")]
  157. pub enum LimiterInferenceKind {
  158. TokenBucket,
  159. FixedWindow,
  160. SlidingWindow,
  161. Unknown,
  162. }
  163. pub fn write_benchmark_report(root: &Path, report: &BenchmarkReport) -> Result<PathBuf> {
  164. write_report(
  165. root,
  166. &report.benchmark,
  167. &report.provider,
  168. &report.model,
  169. report.run.started_at,
  170. report,
  171. )
  172. }
  173. pub fn write_rpm_report(root: &Path, report: &RpmReport) -> Result<PathBuf> {
  174. write_report(
  175. root,
  176. &report.benchmark,
  177. &report.provider,
  178. &report.model,
  179. report.run.started_at,
  180. report,
  181. )
  182. }
  183. fn write_report<T: Serialize>(
  184. root: &Path,
  185. benchmark: &str,
  186. provider: &str,
  187. model: &str,
  188. started_at: DateTime<Utc>,
  189. report: &T,
  190. ) -> Result<PathBuf> {
  191. let reports_dir = root.join("reports");
  192. std::fs::create_dir_all(&reports_dir).with_context(|| {
  193. format!(
  194. "failed to create reports directory {}",
  195. reports_dir.display()
  196. )
  197. })?;
  198. let timestamp = started_at.format("%Y%m%dT%H%M%SZ");
  199. let filename = format!(
  200. "{}-{}-{}-{timestamp}.json",
  201. sanitize_filename_component(benchmark),
  202. sanitize_filename_component(provider),
  203. sanitize_filename_component(model),
  204. );
  205. let path = reports_dir.join(filename);
  206. let contents = serde_json::to_string_pretty(report).context("failed to serialize report")?;
  207. std::fs::write(&path, contents)
  208. .with_context(|| format!("failed to write report {}", path.display()))?;
  209. Ok(path)
  210. }
  211. pub fn sanitize_filename_component(value: &str) -> String {
  212. let sanitized = value
  213. .trim()
  214. .chars()
  215. .map(|ch| {
  216. if ch.is_ascii_alphanumeric() || ch == '.' || ch == '_' || ch == '-' {
  217. ch
  218. } else {
  219. '-'
  220. }
  221. })
  222. .collect::<String>();
  223. let collapsed = sanitized
  224. .split('-')
  225. .filter(|part| !part.is_empty())
  226. .collect::<Vec<_>>()
  227. .join("-");
  228. if collapsed.is_empty() {
  229. "unknown".to_string()
  230. } else {
  231. collapsed
  232. }
  233. }
  234. #[cfg(test)]
  235. mod tests {
  236. use super::*;
  237. use chrono::TimeZone;
  238. #[test]
  239. fn sanitizes_model_names_for_filenames() {
  240. assert_eq!(
  241. sanitize_filename_component("openai/gpt-4.1:mini"),
  242. "openai-gpt-4.1-mini"
  243. );
  244. assert_eq!(sanitize_filename_component(" weird name "), "weird-name");
  245. }
  246. #[test]
  247. fn writes_benchmark_report_under_reports_dir() {
  248. let temp_dir = tempfile::tempdir().expect("create temp dir");
  249. let report = BenchmarkReport {
  250. benchmark: "aime2026".to_string(),
  251. provider: "openai".to_string(),
  252. model: "gpt/test".to_string(),
  253. params: BenchmarkParamsReport {
  254. stream: false,
  255. thinking: None,
  256. },
  257. dataset: DatasetReport {
  258. source: "local".to_string(),
  259. split: "train".to_string(),
  260. revision: None,
  261. local_path: "data/benchmarks/aime2026/aime2026.jsonl".to_string(),
  262. },
  263. run: RunReport {
  264. started_at: Utc.with_ymd_and_hms(2026, 5, 6, 1, 2, 3).unwrap(),
  265. duration_ms: 123,
  266. concurrency: 2,
  267. limit: Some(1),
  268. temperature: 0.0,
  269. max_tokens: 1024,
  270. },
  271. summary: BenchmarkSummaryReport {
  272. accuracy: Some(1.0),
  273. success: 1,
  274. total: 1,
  275. correct: 1,
  276. wrong: 0,
  277. failed: 0,
  278. latency_ms: LatencyReport {
  279. p50: Some(10),
  280. p95: Some(10),
  281. p99: Some(10),
  282. },
  283. ttft_ms: LatencyReport {
  284. p50: None,
  285. p95: None,
  286. p99: None,
  287. },
  288. },
  289. errors: vec![],
  290. correct_samples: vec![],
  291. wrong_cases: vec![],
  292. };
  293. let path = write_benchmark_report(temp_dir.path(), &report).expect("write report");
  294. assert!(path.ends_with("reports/aime2026-openai-gpt-test-20260506T010203Z.json"));
  295. assert!(path.exists());
  296. }
  297. #[test]
  298. fn rpm_report_serializes_params() {
  299. let report = RpmReport {
  300. benchmark: "rpm".to_string(),
  301. provider: "openai".to_string(),
  302. model: "gpt/test".to_string(),
  303. params: RpmParamsReport {
  304. prompt: "Hi".to_string(),
  305. stream: false,
  306. thinking: None,
  307. duration: "60s".to_string(),
  308. burst: None,
  309. concurrency: 10,
  310. probe_seconds: None,
  311. window_offset_ms: 0,
  312. },
  313. run: RpmRunReport {
  314. started_at: Utc.with_ymd_and_hms(2026, 5, 6, 1, 2, 3).unwrap(),
  315. duration_ms: 1000,
  316. target_rpm: 60,
  317. actual_rpm: Some(60.0),
  318. temperature: 0.0,
  319. max_tokens: 1024,
  320. },
  321. summary: RpmSummaryReport {
  322. actual_requests: 1,
  323. success: 1,
  324. failure: 0,
  325. latency_ms: LatencyReport {
  326. p50: Some(10),
  327. p95: Some(10),
  328. p99: Some(10),
  329. },
  330. ttft_ms: LatencyReport {
  331. p50: None,
  332. p95: None,
  333. p99: None,
  334. },
  335. },
  336. mode: "sustained".to_string(),
  337. mode_detail: None,
  338. errors: vec![],
  339. };
  340. let json = serde_json::to_string(&report).expect("serialize report");
  341. assert!(json.contains("\"prompt\":\"Hi\""));
  342. assert!(json.contains("\"stream\":false"));
  343. assert!(json.contains("\"duration\":\"60s\""));
  344. }
  345. #[test]
  346. fn rpm_report_serializes_mode_details_and_inference() {
  347. let report = RpmReport {
  348. benchmark: "rpm".to_string(),
  349. mode: "token-bucket".to_string(),
  350. provider: "anthropic".to_string(),
  351. model: "claude/test".to_string(),
  352. params: RpmParamsReport {
  353. prompt: "Hi".to_string(),
  354. stream: true,
  355. thinking: Some(ThinkingParamsReport {
  356. enabled: true,
  357. kind: Some("enabled".to_string()),
  358. budget_tokens: Some(10000),
  359. effort: None,
  360. display: Some("omitted".to_string()),
  361. reasoning_effort: None,
  362. reasoning_summary: None,
  363. }),
  364. duration: "90s".to_string(),
  365. burst: Some(50),
  366. concurrency: 10,
  367. probe_seconds: Some(90),
  368. window_offset_ms: 200,
  369. },
  370. run: RpmRunReport {
  371. started_at: Utc.with_ymd_and_hms(2026, 5, 6, 1, 2, 3).unwrap(),
  372. duration_ms: 2000,
  373. target_rpm: 120,
  374. actual_rpm: Some(120.0),
  375. temperature: 0.0,
  376. max_tokens: 1024,
  377. },
  378. summary: RpmSummaryReport {
  379. actual_requests: 4,
  380. success: 4,
  381. failure: 0,
  382. latency_ms: LatencyReport {
  383. p50: Some(10),
  384. p95: Some(20),
  385. p99: Some(30),
  386. },
  387. ttft_ms: LatencyReport {
  388. p50: None,
  389. p95: None,
  390. p99: None,
  391. },
  392. },
  393. mode_detail: Some(RpmModeDetailReport {
  394. burst: Some(PhaseSummaryReport {
  395. sent: 2,
  396. success: 2,
  397. failure: 0,
  398. }),
  399. refill_probe: vec![ProbeSecondReport {
  400. second: 1,
  401. sent: 2,
  402. success: 2,
  403. failure: 0,
  404. }],
  405. sliding_probe: vec![],
  406. window_boundary: None,
  407. inference: Some(LimiterInferenceReport {
  408. likely_limiter: LimiterInferenceKind::TokenBucket,
  409. confidence: "medium".to_string(),
  410. signals: vec!["probe success approximated refill".to_string()],
  411. }),
  412. }),
  413. errors: vec![],
  414. };
  415. let json = serde_json::to_string(&report).expect("serialize report");
  416. assert!(json.contains("\"mode\":\"token-bucket\""));
  417. assert!(json.contains("\"refill_probe\""));
  418. assert!(json.contains("\"likely_limiter\":\"token_bucket\""));
  419. }
  420. }