| @@ -418,9 +418,32 @@ fn write_request_error_debug_blocking( | |||||
| mod tests { | mod tests { | ||||
| use crate::runner::{ModelRequest, RawDebugConfig, ThinkingConfig}; | use crate::runner::{ModelRequest, RawDebugConfig, ThinkingConfig}; | ||||
| use reqwest::Client; | use reqwest::Client; | ||||
| use std::path::{Path, PathBuf}; | |||||
| use wiremock::matchers::{body_json, header, method, path}; | use wiremock::matchers::{body_json, header, method, path}; | ||||
| use wiremock::{Mock, MockServer, ResponseTemplate}; | use wiremock::{Mock, MockServer, ResponseTemplate}; | ||||
| fn read_single_debug_file(root: &Path) -> String { | |||||
| let mut files = Vec::new(); | |||||
| collect_debug_files(root, &mut files); | |||||
| assert_eq!(files.len(), 1); | |||||
| std::fs::read_to_string(&files[0]).expect("read raw debug file") | |||||
| } | |||||
| fn collect_debug_files(dir: &Path, files: &mut Vec<PathBuf>) { | |||||
| for entry in std::fs::read_dir(dir) | |||||
| .expect("read debug dir") | |||||
| .collect::<Result<Vec<_>, _>>() | |||||
| .expect("debug entries") | |||||
| { | |||||
| let path = entry.path(); | |||||
| if path.is_dir() { | |||||
| collect_debug_files(&path, files); | |||||
| } else { | |||||
| files.push(path); | |||||
| } | |||||
| } | |||||
| } | |||||
| #[tokio::test] | #[tokio::test] | ||||
| async fn extracts_first_text_block() { | async fn extracts_first_text_block() { | ||||
| let server = MockServer::start().await; | let server = MockServer::start().await; | ||||
| @@ -651,12 +674,7 @@ mod tests { | |||||
| .await | .await | ||||
| .expect_err("non-success should fail"); | .expect_err("non-success should fail"); | ||||
| let debug_files = std::fs::read_dir(temp_dir.path()) | |||||
| .expect("read debug dir") | |||||
| .collect::<Result<Vec<_>, _>>() | |||||
| .expect("debug entries"); | |||||
| assert_eq!(debug_files.len(), 1); | |||||
| let raw = std::fs::read_to_string(debug_files[0].path()).expect("read raw debug file"); | |||||
| let raw = read_single_debug_file(temp_dir.path()); | |||||
| let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | ||||
| assert_eq!(debug["request"]["headers"]["x-api-key"], "[REDACTED]"); | assert_eq!(debug["request"]["headers"]["x-api-key"], "[REDACTED]"); | ||||
| @@ -700,12 +718,7 @@ mod tests { | |||||
| .await | .await | ||||
| .expect_err("connection should fail"); | .expect_err("connection should fail"); | ||||
| let debug_files = std::fs::read_dir(temp_dir.path()) | |||||
| .expect("read debug dir") | |||||
| .collect::<Result<Vec<_>, _>>() | |||||
| .expect("debug entries"); | |||||
| assert_eq!(debug_files.len(), 1); | |||||
| let raw = std::fs::read_to_string(debug_files[0].path()).expect("read raw debug file"); | |||||
| let raw = read_single_debug_file(temp_dir.path()); | |||||
| let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | ||||
| assert_eq!(debug["request"]["method"], "POST"); | assert_eq!(debug["request"]["method"], "POST"); | ||||
| @@ -816,12 +829,7 @@ mod tests { | |||||
| assert_eq!(response.text, "hi there"); | assert_eq!(response.text, "hi there"); | ||||
| assert!(response.first_token_ms.is_some()); | assert!(response.first_token_ms.is_some()); | ||||
| let debug_files = std::fs::read_dir(temp_dir.path()) | |||||
| .expect("read debug dir") | |||||
| .collect::<Result<Vec<_>, _>>() | |||||
| .expect("debug entries"); | |||||
| assert_eq!(debug_files.len(), 1); | |||||
| let raw = std::fs::read_to_string(debug_files[0].path()).expect("read raw debug file"); | |||||
| let raw = read_single_debug_file(temp_dir.path()); | |||||
| assert!(raw.contains("event: content_block_delta")); | assert!(raw.contains("event: content_block_delta")); | ||||
| assert!(raw.contains("\"text\":\"hi \"")); | assert!(raw.contains("\"text\":\"hi \"")); | ||||
| } | } | ||||
| @@ -872,12 +880,7 @@ mod tests { | |||||
| .contains("completed without producing any content") | .contains("completed without producing any content") | ||||
| ); | ); | ||||
| let debug_files = std::fs::read_dir(temp_dir.path()) | |||||
| .expect("read debug dir") | |||||
| .collect::<Result<Vec<_>, _>>() | |||||
| .expect("debug entries"); | |||||
| assert_eq!(debug_files.len(), 1); | |||||
| let raw = std::fs::read_to_string(debug_files[0].path()).expect("read raw debug file"); | |||||
| let raw = read_single_debug_file(temp_dir.path()); | |||||
| let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | ||||
| assert_eq!( | assert_eq!( | ||||
| @@ -951,12 +954,7 @@ mod tests { | |||||
| server.await.expect("server task"); | server.await.expect("server task"); | ||||
| assert!(error.to_string().contains("stream interrupted")); | assert!(error.to_string().contains("stream interrupted")); | ||||
| let debug_files = std::fs::read_dir(temp_dir.path()) | |||||
| .expect("read debug dir") | |||||
| .collect::<Result<Vec<_>, _>>() | |||||
| .expect("debug entries"); | |||||
| assert_eq!(debug_files.len(), 1); | |||||
| let raw = std::fs::read_to_string(debug_files[0].path()).expect("read raw debug file"); | |||||
| let raw = read_single_debug_file(temp_dir.path()); | |||||
| let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | ||||
| assert_eq!(debug["response"]["status"], 200); | assert_eq!(debug["response"]["status"], 200); | ||||
| @@ -415,9 +415,32 @@ struct GooglePart { | |||||
| mod tests { | mod tests { | ||||
| use crate::runner::{ModelRequest, RawDebugConfig, ThinkingConfig}; | use crate::runner::{ModelRequest, RawDebugConfig, ThinkingConfig}; | ||||
| use reqwest::Client; | use reqwest::Client; | ||||
| use std::path::{Path, PathBuf}; | |||||
| use wiremock::matchers::{body_json, header, method, path}; | use wiremock::matchers::{body_json, header, method, path}; | ||||
| use wiremock::{Mock, MockServer, ResponseTemplate}; | use wiremock::{Mock, MockServer, ResponseTemplate}; | ||||
| fn read_single_debug_file(root: &Path) -> String { | |||||
| let mut files = Vec::new(); | |||||
| collect_debug_files(root, &mut files); | |||||
| assert_eq!(files.len(), 1); | |||||
| std::fs::read_to_string(&files[0]).expect("read raw debug file") | |||||
| } | |||||
| fn collect_debug_files(dir: &Path, files: &mut Vec<PathBuf>) { | |||||
| for entry in std::fs::read_dir(dir) | |||||
| .expect("read debug dir") | |||||
| .collect::<Result<Vec<_>, _>>() | |||||
| .expect("debug entries") | |||||
| { | |||||
| let path = entry.path(); | |||||
| if path.is_dir() { | |||||
| collect_debug_files(&path, files); | |||||
| } else { | |||||
| files.push(path); | |||||
| } | |||||
| } | |||||
| } | |||||
| #[tokio::test] | #[tokio::test] | ||||
| async fn sends_generate_content_with_thinking_config_and_extracts_text() { | async fn sends_generate_content_with_thinking_config_and_extracts_text() { | ||||
| let server = MockServer::start().await; | let server = MockServer::start().await; | ||||
| @@ -560,12 +583,7 @@ mod tests { | |||||
| .await | .await | ||||
| .expect_err("non-success should fail"); | .expect_err("non-success should fail"); | ||||
| let debug_files = std::fs::read_dir(temp_dir.path()) | |||||
| .expect("read debug dir") | |||||
| .collect::<Result<Vec<_>, _>>() | |||||
| .expect("debug entries"); | |||||
| assert_eq!(debug_files.len(), 1); | |||||
| let raw = std::fs::read_to_string(debug_files[0].path()).expect("read raw debug file"); | |||||
| let raw = read_single_debug_file(temp_dir.path()); | |||||
| let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | ||||
| assert_eq!(debug["request"]["headers"]["x-goog-api-key"], "[REDACTED]"); | assert_eq!(debug["request"]["headers"]["x-goog-api-key"], "[REDACTED]"); | ||||
| @@ -606,12 +624,7 @@ mod tests { | |||||
| .await | .await | ||||
| .expect_err("connection should fail"); | .expect_err("connection should fail"); | ||||
| let debug_files = std::fs::read_dir(temp_dir.path()) | |||||
| .expect("read debug dir") | |||||
| .collect::<Result<Vec<_>, _>>() | |||||
| .expect("debug entries"); | |||||
| assert_eq!(debug_files.len(), 1); | |||||
| let raw = std::fs::read_to_string(debug_files[0].path()).expect("read raw debug file"); | |||||
| let raw = read_single_debug_file(temp_dir.path()); | |||||
| let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | ||||
| assert_eq!(debug["request"]["method"], "POST"); | assert_eq!(debug["request"]["method"], "POST"); | ||||
| @@ -403,9 +403,32 @@ fn write_request_error_debug_blocking( | |||||
| mod tests { | mod tests { | ||||
| use crate::runner::{ModelRequest, RawDebugConfig, ThinkingConfig}; | use crate::runner::{ModelRequest, RawDebugConfig, ThinkingConfig}; | ||||
| use reqwest::Client; | use reqwest::Client; | ||||
| use std::path::{Path, PathBuf}; | |||||
| use wiremock::matchers::{body_json, header, method, path}; | use wiremock::matchers::{body_json, header, method, path}; | ||||
| use wiremock::{Mock, MockServer, ResponseTemplate}; | use wiremock::{Mock, MockServer, ResponseTemplate}; | ||||
| fn read_single_debug_file(root: &Path) -> String { | |||||
| let mut files = Vec::new(); | |||||
| collect_debug_files(root, &mut files); | |||||
| assert_eq!(files.len(), 1); | |||||
| std::fs::read_to_string(&files[0]).expect("read raw debug file") | |||||
| } | |||||
| fn collect_debug_files(dir: &Path, files: &mut Vec<PathBuf>) { | |||||
| for entry in std::fs::read_dir(dir) | |||||
| .expect("read debug dir") | |||||
| .collect::<Result<Vec<_>, _>>() | |||||
| .expect("debug entries") | |||||
| { | |||||
| let path = entry.path(); | |||||
| if path.is_dir() { | |||||
| collect_debug_files(&path, files); | |||||
| } else { | |||||
| files.push(path); | |||||
| } | |||||
| } | |||||
| } | |||||
| #[tokio::test] | #[tokio::test] | ||||
| async fn extracts_chat_completion_text() { | async fn extracts_chat_completion_text() { | ||||
| let server = MockServer::start().await; | let server = MockServer::start().await; | ||||
| @@ -595,12 +618,7 @@ mod tests { | |||||
| .await | .await | ||||
| .expect_err("non-success should fail"); | .expect_err("non-success should fail"); | ||||
| let debug_files = std::fs::read_dir(temp_dir.path()) | |||||
| .expect("read debug dir") | |||||
| .collect::<Result<Vec<_>, _>>() | |||||
| .expect("debug entries"); | |||||
| assert_eq!(debug_files.len(), 1); | |||||
| let raw = std::fs::read_to_string(debug_files[0].path()).expect("read raw debug file"); | |||||
| let raw = read_single_debug_file(temp_dir.path()); | |||||
| let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | ||||
| assert_eq!(debug["request"]["method"], "POST"); | assert_eq!(debug["request"]["method"], "POST"); | ||||
| @@ -651,12 +669,7 @@ mod tests { | |||||
| .await | .await | ||||
| .expect_err("connection should fail"); | .expect_err("connection should fail"); | ||||
| let debug_files = std::fs::read_dir(temp_dir.path()) | |||||
| .expect("read debug dir") | |||||
| .collect::<Result<Vec<_>, _>>() | |||||
| .expect("debug entries"); | |||||
| assert_eq!(debug_files.len(), 1); | |||||
| let raw = std::fs::read_to_string(debug_files[0].path()).expect("read raw debug file"); | |||||
| let raw = read_single_debug_file(temp_dir.path()); | |||||
| let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json"); | ||||
| assert_eq!(debug["request"]["method"], "POST"); | assert_eq!(debug["request"]["method"], "POST"); | ||||
| @@ -768,12 +781,7 @@ mod tests { | |||||
| assert_eq!(response.text, "hi there"); | assert_eq!(response.text, "hi there"); | ||||
| assert!(response.first_token_ms.is_some()); | assert!(response.first_token_ms.is_some()); | ||||
| let debug_files = std::fs::read_dir(temp_dir.path()) | |||||
| .expect("read debug dir") | |||||
| .collect::<Result<Vec<_>, _>>() | |||||
| .expect("debug entries"); | |||||
| assert_eq!(debug_files.len(), 1); | |||||
| let raw = std::fs::read_to_string(debug_files[0].path()).expect("read raw debug file"); | |||||
| let raw = read_single_debug_file(temp_dir.path()); | |||||
| assert!(raw.contains("data: {\"choices\"")); | assert!(raw.contains("data: {\"choices\"")); | ||||
| assert!(raw.contains("data: [DONE]")); | assert!(raw.contains("data: [DONE]")); | ||||
| } | } | ||||
| @@ -102,23 +102,25 @@ impl RawDebugConfig { | |||||
| } | } | ||||
| async fn write_debug_file(&self, response_kind: &str, contents: &str) -> Result<PathBuf> { | async fn write_debug_file(&self, response_kind: &str, contents: &str) -> Result<PathBuf> { | ||||
| tokio::fs::create_dir_all(&self.output_dir) | |||||
| let now = Utc::now(); | |||||
| let dated_output_dir = self.output_dir.join(now.format("%Y%m%d").to_string()); | |||||
| tokio::fs::create_dir_all(&dated_output_dir) | |||||
| .await | .await | ||||
| .with_context(|| { | .with_context(|| { | ||||
| format!( | format!( | ||||
| "failed to create raw debug dir {}", | "failed to create raw debug dir {}", | ||||
| self.output_dir.display() | |||||
| dated_output_dir.display() | |||||
| ) | ) | ||||
| })?; | })?; | ||||
| let sequence = self.counter.fetch_add(1, Ordering::Relaxed) + 1; | let sequence = self.counter.fetch_add(1, Ordering::Relaxed) + 1; | ||||
| let timestamp = Utc::now().format("%Y%m%dT%H%M%S%.3fZ"); | |||||
| let timestamp = now.format("%Y%m%dT%H%M%S%.3fZ"); | |||||
| let filename = format!( | let filename = format!( | ||||
| "{}-{}-{sequence:06}-{}.txt", | "{}-{}-{sequence:06}-{}.txt", | ||||
| self.prefix, | self.prefix, | ||||
| timestamp, | timestamp, | ||||
| sanitize_filename_component(response_kind) | sanitize_filename_component(response_kind) | ||||
| ); | ); | ||||
| let path = self.output_dir.join(filename); | |||||
| let path = dated_output_dir.join(filename); | |||||
| tokio::fs::write(&path, contents) | tokio::fs::write(&path, contents) | ||||
| .await | .await | ||||
| .with_context(|| format!("failed to write raw debug response {}", path.display()))?; | .with_context(|| format!("failed to write raw debug response {}", path.display()))?; | ||||
| @@ -298,4 +300,28 @@ mod tests { | |||||
| assert!(debug.contains("[REDACTED]")); | assert!(debug.contains("[REDACTED]")); | ||||
| assert!(!debug.contains("sk-secret-token")); | assert!(!debug.contains("sk-secret-token")); | ||||
| } | } | ||||
| #[tokio::test] | |||||
| async fn raw_debug_writes_files_under_date_directory() { | |||||
| let temp_dir = tempfile::tempdir().expect("create temp dir"); | |||||
| let raw_debug = RawDebugConfig::new(temp_dir.path().join("debug"), "model/test".into()); | |||||
| let path = raw_debug | |||||
| .write_response("response-kind", "body") | |||||
| .await | |||||
| .expect("write debug response"); | |||||
| let relative = path | |||||
| .strip_prefix(temp_dir.path().join("debug")) | |||||
| .expect("path under debug dir"); | |||||
| let components = relative | |||||
| .components() | |||||
| .map(|component| component.as_os_str().to_string_lossy().into_owned()) | |||||
| .collect::<Vec<_>>(); | |||||
| assert_eq!(components.len(), 2); | |||||
| assert_eq!(components[0].len(), 8); | |||||
| assert!(components[0].chars().all(|ch| ch.is_ascii_digit())); | |||||
| assert!(components[1].ends_with("-response-kind.txt")); | |||||
| } | |||||
| } | } | ||||