api模型检测
Nelze vybrat více než 25 témat Téma musí začínat písmenem nebo číslem, může obsahovat pomlčky („-“) a může být dlouhé až 35 znaků.
 
 

232 řádky
6.0 KiB

  1. use hdrhistogram::Histogram;
  2. use serde::Serialize;
  3. use std::collections::BTreeMap;
  4. #[derive(Debug)]
  5. pub struct Metrics {
  6. success: u64,
  7. failed: u64,
  8. correct: u64,
  9. total_judged: u64,
  10. errors: BTreeMap<String, u64>,
  11. latency: Histogram<u64>,
  12. ttft: Histogram<u64>,
  13. }
  14. #[derive(Debug, Clone, PartialEq, Serialize)]
  15. pub struct MetricsSummary {
  16. pub success: u64,
  17. pub failed: u64,
  18. pub total: u64,
  19. pub correct: u64,
  20. pub wrong: u64,
  21. pub total_judged: u64,
  22. pub accuracy: Option<f64>,
  23. pub latency_ms: LatencySummary,
  24. pub ttft_ms: LatencySummary,
  25. pub errors: Vec<ErrorCount>,
  26. }
  27. #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
  28. pub struct LatencySummary {
  29. pub p50: Option<u64>,
  30. pub p95: Option<u64>,
  31. pub p99: Option<u64>,
  32. }
  33. #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
  34. pub struct ErrorCount {
  35. pub code: String,
  36. pub count: u64,
  37. }
  38. impl ErrorCount {
  39. pub fn new(code: impl Into<String>, count: u64) -> Self {
  40. Self {
  41. code: code.into(),
  42. count,
  43. }
  44. }
  45. }
  46. impl Metrics {
  47. pub fn new() -> Self {
  48. Self {
  49. success: 0,
  50. failed: 0,
  51. correct: 0,
  52. total_judged: 0,
  53. errors: BTreeMap::new(),
  54. latency: Histogram::new(3).expect("valid histogram precision"),
  55. ttft: Histogram::new(3).expect("valid histogram precision"),
  56. }
  57. }
  58. pub fn record_success(&mut self, _status: u16, latency_ms: u64, first_token_ms: Option<u64>) {
  59. self.success += 1;
  60. let _ = self.latency.record(latency_ms);
  61. if let Some(ttft) = first_token_ms {
  62. let _ = self.ttft.record(ttft);
  63. }
  64. }
  65. pub fn record_failure(&mut self, code: impl Into<String>) {
  66. self.failed += 1;
  67. *self.errors.entry(code.into()).or_insert(0) += 1;
  68. }
  69. pub fn record_judgement(&mut self, correct: bool) {
  70. self.total_judged += 1;
  71. if correct {
  72. self.correct += 1;
  73. }
  74. }
  75. pub fn summary(&self) -> MetricsSummary {
  76. let wrong = self.total_judged.saturating_sub(self.correct);
  77. let accuracy = if self.total_judged == 0 {
  78. None
  79. } else {
  80. Some(self.correct as f64 / self.total_judged as f64)
  81. };
  82. let latency_ms = if self.latency.is_empty() {
  83. LatencySummary {
  84. p50: None,
  85. p95: None,
  86. p99: None,
  87. }
  88. } else {
  89. LatencySummary {
  90. p50: Some(self.latency.value_at_quantile(0.50)),
  91. p95: Some(self.latency.value_at_quantile(0.95)),
  92. p99: Some(self.latency.value_at_quantile(0.99)),
  93. }
  94. };
  95. let ttft_ms = if self.ttft.is_empty() {
  96. LatencySummary {
  97. p50: None,
  98. p95: None,
  99. p99: None,
  100. }
  101. } else {
  102. LatencySummary {
  103. p50: Some(self.ttft.value_at_quantile(0.50)),
  104. p95: Some(self.ttft.value_at_quantile(0.95)),
  105. p99: Some(self.ttft.value_at_quantile(0.99)),
  106. }
  107. };
  108. MetricsSummary {
  109. success: self.success,
  110. failed: self.failed,
  111. total: self.success + self.failed,
  112. correct: self.correct,
  113. wrong,
  114. total_judged: self.total_judged,
  115. accuracy,
  116. latency_ms,
  117. ttft_ms,
  118. errors: self
  119. .errors
  120. .iter()
  121. .map(|(code, count)| ErrorCount::new(code.clone(), *count))
  122. .collect(),
  123. }
  124. }
  125. }
  126. impl Default for Metrics {
  127. fn default() -> Self {
  128. Self::new()
  129. }
  130. }
  131. #[cfg(test)]
  132. mod tests {
  133. use super::*;
  134. #[test]
  135. fn tracks_success_failure_and_error_counts() {
  136. let mut metrics = Metrics::new();
  137. metrics.record_success(200, 120, None);
  138. metrics.record_failure("request_error");
  139. metrics.record_failure("429");
  140. metrics.record_failure("429");
  141. let summary = metrics.summary();
  142. assert_eq!(summary.success, 1);
  143. assert_eq!(summary.failed, 3);
  144. assert_eq!(summary.total, 4);
  145. assert_eq!(
  146. summary.errors,
  147. vec![
  148. ErrorCount::new("429", 2),
  149. ErrorCount::new("request_error", 1)
  150. ]
  151. );
  152. }
  153. #[test]
  154. fn computes_accuracy_from_judged_cases() {
  155. let mut metrics = Metrics::new();
  156. metrics.record_judgement(true);
  157. metrics.record_judgement(false);
  158. metrics.record_judgement(true);
  159. let summary = metrics.summary();
  160. assert_eq!(summary.correct, 2);
  161. assert_eq!(summary.wrong, 1);
  162. assert_eq!(summary.total_judged, 3);
  163. assert_eq!(summary.accuracy, Some(2.0 / 3.0));
  164. }
  165. #[test]
  166. fn computes_latency_percentiles() {
  167. let mut metrics = Metrics::new();
  168. for latency in 1..=100 {
  169. metrics.record_success(200, latency, None);
  170. }
  171. let summary = metrics.summary();
  172. assert_eq!(summary.latency_ms.p50, Some(50));
  173. assert_eq!(summary.latency_ms.p95, Some(95));
  174. assert_eq!(summary.latency_ms.p99, Some(99));
  175. }
  176. #[test]
  177. fn computes_ttft_percentiles_when_present() {
  178. let mut metrics = Metrics::new();
  179. for i in 1..=100 {
  180. metrics.record_success(200, i * 2, Some(i));
  181. }
  182. let summary = metrics.summary();
  183. assert_eq!(summary.ttft_ms.p50, Some(50));
  184. assert_eq!(summary.ttft_ms.p95, Some(95));
  185. assert_eq!(summary.ttft_ms.p99, Some(99));
  186. }
  187. #[test]
  188. fn ttft_is_none_when_no_streaming_requests() {
  189. let mut metrics = Metrics::new();
  190. metrics.record_success(200, 100, None);
  191. metrics.record_success(200, 200, None);
  192. let summary = metrics.summary();
  193. assert_eq!(summary.ttft_ms.p50, None);
  194. assert_eq!(summary.ttft_ms.p95, None);
  195. assert_eq!(summary.ttft_ms.p99, None);
  196. }
  197. }