|
- use hdrhistogram::Histogram;
- use serde::Serialize;
- use std::collections::BTreeMap;
-
- #[derive(Debug)]
- pub struct Metrics {
- success: u64,
- failed: u64,
- correct: u64,
- total_judged: u64,
- errors: BTreeMap<String, u64>,
- latency: Histogram<u64>,
- ttft: Histogram<u64>,
- }
-
- #[derive(Debug, Clone, PartialEq, Serialize)]
- pub struct MetricsSummary {
- pub success: u64,
- pub failed: u64,
- pub total: u64,
- pub correct: u64,
- pub wrong: u64,
- pub total_judged: u64,
- pub accuracy: Option<f64>,
- pub latency_ms: LatencySummary,
- pub ttft_ms: LatencySummary,
- pub errors: Vec<ErrorCount>,
- }
-
- #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
- pub struct LatencySummary {
- pub p50: Option<u64>,
- pub p95: Option<u64>,
- pub p99: Option<u64>,
- }
-
- #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
- pub struct ErrorCount {
- pub code: String,
- pub count: u64,
- }
-
- impl ErrorCount {
- pub fn new(code: impl Into<String>, count: u64) -> Self {
- Self {
- code: code.into(),
- count,
- }
- }
- }
-
- impl Metrics {
- pub fn new() -> Self {
- Self {
- success: 0,
- failed: 0,
- correct: 0,
- total_judged: 0,
- errors: BTreeMap::new(),
- latency: Histogram::new(3).expect("valid histogram precision"),
- ttft: Histogram::new(3).expect("valid histogram precision"),
- }
- }
-
- pub fn record_success(&mut self, _status: u16, latency_ms: u64, first_token_ms: Option<u64>) {
- self.success += 1;
- let _ = self.latency.record(latency_ms);
- if let Some(ttft) = first_token_ms {
- let _ = self.ttft.record(ttft);
- }
- }
-
- pub fn record_failure(&mut self, code: impl Into<String>) {
- self.failed += 1;
- *self.errors.entry(code.into()).or_insert(0) += 1;
- }
-
- pub fn record_judgement(&mut self, correct: bool) {
- self.total_judged += 1;
- if correct {
- self.correct += 1;
- }
- }
-
- pub fn summary(&self) -> MetricsSummary {
- let wrong = self.total_judged.saturating_sub(self.correct);
- let accuracy = if self.total_judged == 0 {
- None
- } else {
- Some(self.correct as f64 / self.total_judged as f64)
- };
- let latency_ms = if self.latency.is_empty() {
- LatencySummary {
- p50: None,
- p95: None,
- p99: None,
- }
- } else {
- LatencySummary {
- p50: Some(self.latency.value_at_quantile(0.50)),
- p95: Some(self.latency.value_at_quantile(0.95)),
- p99: Some(self.latency.value_at_quantile(0.99)),
- }
- };
-
- let ttft_ms = if self.ttft.is_empty() {
- LatencySummary {
- p50: None,
- p95: None,
- p99: None,
- }
- } else {
- LatencySummary {
- p50: Some(self.ttft.value_at_quantile(0.50)),
- p95: Some(self.ttft.value_at_quantile(0.95)),
- p99: Some(self.ttft.value_at_quantile(0.99)),
- }
- };
-
- MetricsSummary {
- success: self.success,
- failed: self.failed,
- total: self.success + self.failed,
- correct: self.correct,
- wrong,
- total_judged: self.total_judged,
- accuracy,
- latency_ms,
- ttft_ms,
- errors: self
- .errors
- .iter()
- .map(|(code, count)| ErrorCount::new(code.clone(), *count))
- .collect(),
- }
- }
- }
-
- impl Default for Metrics {
- fn default() -> Self {
- Self::new()
- }
- }
-
- #[cfg(test)]
- mod tests {
- use super::*;
-
- #[test]
- fn tracks_success_failure_and_error_counts() {
- let mut metrics = Metrics::new();
-
- metrics.record_success(200, 120, None);
- metrics.record_failure("request_error");
- metrics.record_failure("429");
- metrics.record_failure("429");
-
- let summary = metrics.summary();
-
- assert_eq!(summary.success, 1);
- assert_eq!(summary.failed, 3);
- assert_eq!(summary.total, 4);
- assert_eq!(
- summary.errors,
- vec![
- ErrorCount::new("429", 2),
- ErrorCount::new("request_error", 1)
- ]
- );
- }
-
- #[test]
- fn computes_accuracy_from_judged_cases() {
- let mut metrics = Metrics::new();
-
- metrics.record_judgement(true);
- metrics.record_judgement(false);
- metrics.record_judgement(true);
-
- let summary = metrics.summary();
-
- assert_eq!(summary.correct, 2);
- assert_eq!(summary.wrong, 1);
- assert_eq!(summary.total_judged, 3);
- assert_eq!(summary.accuracy, Some(2.0 / 3.0));
- }
-
- #[test]
- fn computes_latency_percentiles() {
- let mut metrics = Metrics::new();
-
- for latency in 1..=100 {
- metrics.record_success(200, latency, None);
- }
-
- let summary = metrics.summary();
-
- assert_eq!(summary.latency_ms.p50, Some(50));
- assert_eq!(summary.latency_ms.p95, Some(95));
- assert_eq!(summary.latency_ms.p99, Some(99));
- }
-
- #[test]
- fn computes_ttft_percentiles_when_present() {
- let mut metrics = Metrics::new();
-
- for i in 1..=100 {
- metrics.record_success(200, i * 2, Some(i));
- }
-
- let summary = metrics.summary();
-
- assert_eq!(summary.ttft_ms.p50, Some(50));
- assert_eq!(summary.ttft_ms.p95, Some(95));
- assert_eq!(summary.ttft_ms.p99, Some(99));
- }
-
- #[test]
- fn ttft_is_none_when_no_streaming_requests() {
- let mut metrics = Metrics::new();
-
- metrics.record_success(200, 100, None);
- metrics.record_success(200, 200, None);
-
- let summary = metrics.summary();
-
- assert_eq!(summary.ttft_ms.p50, None);
- assert_eq!(summary.ttft_ms.p95, None);
- assert_eq!(summary.ttft_ms.p99, None);
- }
- }
|