api模型检测
Вы не можете выбрать более 25 тем Темы должны начинаться с буквы или цифры, могут содержать дефисы(-) и должны содержать не более 35 символов.

1000 строки
33 KiB

  1. use crate::runner::{
  2. HttpDebugRequest, HttpDebugResponse, ModelRequest, ModelResponse, request_headers_for_debug,
  3. response_headers_for_debug,
  4. };
  5. use anyhow::{Context, Result, bail};
  6. use futures::StreamExt;
  7. use reqwest::Client;
  8. use serde::Deserialize;
  9. use serde_json::{Value, json};
  10. use std::time::Instant;
  11. pub async fn send(client: &Client, request: &ModelRequest) -> Result<ModelResponse> {
  12. let url = super::endpoint_url(&request.base_url, "/v1/messages")?;
  13. let debug_url = url.to_string();
  14. let request_body = request_body(request, false);
  15. let response = client
  16. .post(url)
  17. .header("x-api-key", &request.api_token)
  18. .header("anthropic-version", "2023-06-01")
  19. .json(&request_body)
  20. .send()
  21. .await
  22. .map_err(|error| {
  23. write_request_error_debug_blocking(
  24. request,
  25. &debug_url,
  26. request_body.clone(),
  27. format!("failed to send Anthropic messages request: {error}"),
  28. "anthropic-request-error",
  29. );
  30. error
  31. })
  32. .context("failed to send Anthropic messages request")?;
  33. let status = response.status();
  34. let status_code = status.as_u16();
  35. let response_headers = response.headers().clone();
  36. let body = response
  37. .text()
  38. .await
  39. .map_err(|error| {
  40. write_request_error_debug_blocking(
  41. request,
  42. &debug_url,
  43. request_body.clone(),
  44. format!("failed to read Anthropic response body: {error}"),
  45. "anthropic-request-error",
  46. );
  47. error
  48. })
  49. .context("failed to read Anthropic response body")?;
  50. if status.is_success()
  51. && let Some(raw_debug) = &request.raw_debug
  52. && raw_debug.write_success_raw()
  53. {
  54. raw_debug
  55. .write_response("anthropic-json", &body)
  56. .await
  57. .context("failed to write Anthropic raw debug response")?;
  58. }
  59. if !status.is_success() {
  60. write_error_debug(
  61. request,
  62. &debug_url,
  63. request_body,
  64. status_code,
  65. &response_headers,
  66. &body,
  67. "anthropic-error-http",
  68. )
  69. .await?;
  70. bail!(
  71. "{}",
  72. super::upstream_error_message("Anthropic", status_code, &body)
  73. );
  74. }
  75. let parsed: AnthropicResponse = match serde_json::from_str(&body) {
  76. Ok(parsed) => parsed,
  77. Err(error) => {
  78. write_response_request_error_debug(
  79. request,
  80. &debug_url,
  81. request_body,
  82. status_code,
  83. &response_headers,
  84. &body,
  85. format!("failed to parse Anthropic response JSON: {error}"),
  86. )
  87. .await?;
  88. return Err(error).context("failed to parse Anthropic response JSON");
  89. }
  90. };
  91. let Some(text) = parsed.content.into_iter().find_map(|block| match block {
  92. AnthropicContentBlock::Text { text } if !text.is_empty() => Some(text),
  93. _ => None,
  94. }) else {
  95. write_response_request_error_debug(
  96. request,
  97. &debug_url,
  98. request_body,
  99. status_code,
  100. &response_headers,
  101. &body,
  102. "Anthropic response missing first text content block".to_string(),
  103. )
  104. .await?;
  105. bail!("Anthropic response missing first text content block");
  106. };
  107. Ok(ModelResponse {
  108. text,
  109. status: status_code,
  110. elapsed_ms: 0,
  111. first_token_ms: None,
  112. chunk_elapsed_ms: Vec::new(),
  113. })
  114. }
  115. pub async fn send_stream(client: &Client, request: &ModelRequest) -> Result<ModelResponse> {
  116. let url = super::endpoint_url(&request.base_url, "/v1/messages")?;
  117. let debug_url = url.to_string();
  118. let started = Instant::now();
  119. let request_body = request_body(request, true);
  120. let response = client
  121. .post(url)
  122. .header("x-api-key", &request.api_token)
  123. .header("anthropic-version", "2023-06-01")
  124. .json(&request_body)
  125. .send()
  126. .await
  127. .map_err(|error| {
  128. write_request_error_debug_blocking(
  129. request,
  130. &debug_url,
  131. request_body.clone(),
  132. format!("failed to send Anthropic streaming request: {error}"),
  133. "anthropic-request-error",
  134. );
  135. error
  136. })
  137. .context("failed to send Anthropic streaming request")?;
  138. let status = response.status();
  139. let status_code = status.as_u16();
  140. let response_headers = response.headers().clone();
  141. if !status.is_success() {
  142. let body = response
  143. .text()
  144. .await
  145. .map_err(|error| {
  146. write_request_error_debug_blocking(
  147. request,
  148. &debug_url,
  149. request_body.clone(),
  150. format!("failed to read Anthropic error response body: {error}"),
  151. "anthropic-request-error",
  152. );
  153. error
  154. })
  155. .context("failed to read Anthropic error response body")?;
  156. if let Some(raw_debug) = &request.raw_debug {
  157. raw_debug
  158. .write_http_error(
  159. "anthropic-error-http",
  160. debug_request(request, &debug_url, request_body),
  161. HttpDebugResponse {
  162. status: Some(status_code),
  163. headers: response_headers_for_debug(&response_headers),
  164. body: body.clone(),
  165. error_kind: None,
  166. error: None,
  167. },
  168. )
  169. .await
  170. .context("failed to write Anthropic raw debug error response")?;
  171. }
  172. bail!(
  173. "{}",
  174. super::upstream_error_message("Anthropic", status_code, &body)
  175. );
  176. }
  177. let mut stream = response.bytes_stream();
  178. let mut buffer = super::SseLineBuffer::new();
  179. let mut text = String::new();
  180. let mut raw_stream = String::new();
  181. let mut first_token_ms: Option<u128> = None;
  182. let mut chunk_elapsed_ms = Vec::new();
  183. let mut current_event = String::new();
  184. let mut done = false;
  185. while let Some(chunk) = stream.next().await {
  186. let chunk = match chunk {
  187. Ok(chunk) => chunk,
  188. Err(error) => {
  189. write_response_request_error_debug(
  190. request,
  191. &debug_url,
  192. request_body.clone(),
  193. status_code,
  194. &response_headers,
  195. &raw_stream,
  196. format!("Anthropic stream interrupted: {error}"),
  197. )
  198. .await?;
  199. return Err(error).context("Anthropic stream interrupted");
  200. }
  201. };
  202. chunk_elapsed_ms.push(started.elapsed().as_millis());
  203. for line in buffer.feed(&chunk) {
  204. raw_stream.push_str(&line);
  205. raw_stream.push('\n');
  206. if let Some(event_type) = line.strip_prefix("event: ") {
  207. current_event = event_type.to_string();
  208. continue;
  209. }
  210. if let Some(data) = line.strip_prefix("data: ") {
  211. if current_event == "content_block_delta" {
  212. if let Some(content) = serde_json::from_str::<AnthropicStreamDelta>(data)
  213. .ok()
  214. .and_then(|p| p.delta.text)
  215. .filter(|c| !c.is_empty())
  216. {
  217. if first_token_ms.is_none() {
  218. first_token_ms = Some(started.elapsed().as_millis());
  219. }
  220. text.push_str(&content);
  221. }
  222. } else if current_event == "message_stop" {
  223. done = true;
  224. break;
  225. }
  226. }
  227. if line.is_empty() {
  228. current_event.clear();
  229. }
  230. }
  231. if done {
  232. break;
  233. }
  234. }
  235. if let Some(raw_debug) = &request.raw_debug
  236. && raw_debug.write_success_raw()
  237. {
  238. raw_debug
  239. .write_response("anthropic-sse", &raw_stream)
  240. .await
  241. .context("failed to write Anthropic raw debug stream")?;
  242. }
  243. if text.is_empty() {
  244. write_response_request_error_debug(
  245. request,
  246. &debug_url,
  247. request_body,
  248. status_code,
  249. &response_headers,
  250. &raw_stream,
  251. "Anthropic stream completed without producing any content".to_string(),
  252. )
  253. .await?;
  254. bail!("Anthropic stream completed without producing any content");
  255. }
  256. Ok(ModelResponse {
  257. text,
  258. status: status_code,
  259. elapsed_ms: 0,
  260. first_token_ms,
  261. chunk_elapsed_ms,
  262. })
  263. }
  264. #[derive(Debug, Deserialize)]
  265. struct AnthropicResponse {
  266. content: Vec<AnthropicContentBlock>,
  267. }
  268. #[derive(Debug, Deserialize)]
  269. #[serde(tag = "type")]
  270. enum AnthropicContentBlock {
  271. #[serde(rename = "text")]
  272. Text { text: String },
  273. #[serde(other)]
  274. Other,
  275. }
  276. #[derive(Debug, Deserialize)]
  277. struct AnthropicStreamDelta {
  278. delta: AnthropicDeltaContent,
  279. }
  280. #[derive(Debug, Deserialize)]
  281. struct AnthropicDeltaContent {
  282. text: Option<String>,
  283. }
  284. fn request_body(request: &ModelRequest, stream: bool) -> Value {
  285. let mut body = json!({
  286. "model": request.model,
  287. "messages": [{"role": "user", "content": request.prompt}],
  288. "temperature": request.temperature,
  289. "max_tokens": request.max_tokens
  290. });
  291. let object = body
  292. .as_object_mut()
  293. .expect("Anthropic request body is an object");
  294. if stream {
  295. object.insert("stream".to_string(), json!(true));
  296. }
  297. if let Some(thinking) = &request.thinking
  298. && thinking.enabled
  299. {
  300. object.remove("temperature");
  301. let mut thinking_body = serde_json::Map::new();
  302. let thinking_type = thinking.kind.as_deref().unwrap_or("enabled");
  303. thinking_body.insert("type".to_string(), json!(thinking_type));
  304. if thinking_type == "adaptive" {
  305. if let Some(effort) = &thinking.effort {
  306. thinking_body.insert("effort".to_string(), json!(effort));
  307. }
  308. } else if let Some(budget_tokens) = thinking.budget_tokens {
  309. thinking_body.insert("budget_tokens".to_string(), json!(budget_tokens));
  310. }
  311. if let Some(display) = &thinking.display {
  312. thinking_body.insert("display".to_string(), json!(display));
  313. }
  314. object.insert("thinking".to_string(), Value::Object(thinking_body));
  315. }
  316. body
  317. }
  318. async fn write_error_debug(
  319. request: &ModelRequest,
  320. url: &str,
  321. request_body: Value,
  322. status: u16,
  323. response_headers: &reqwest::header::HeaderMap,
  324. response_body: &str,
  325. response_kind: &str,
  326. ) -> Result<()> {
  327. if let Some(raw_debug) = &request.raw_debug {
  328. raw_debug
  329. .write_http_error(
  330. response_kind,
  331. debug_request(request, url, request_body),
  332. HttpDebugResponse {
  333. status: Some(status),
  334. headers: response_headers_for_debug(response_headers),
  335. body: response_body.to_string(),
  336. error_kind: None,
  337. error: None,
  338. },
  339. )
  340. .await
  341. .context("failed to write Anthropic raw debug error response")?;
  342. }
  343. Ok(())
  344. }
  345. async fn write_response_request_error_debug(
  346. request: &ModelRequest,
  347. url: &str,
  348. request_body: Value,
  349. status: u16,
  350. response_headers: &reqwest::header::HeaderMap,
  351. response_body: &str,
  352. error: String,
  353. ) -> Result<()> {
  354. if let Some(raw_debug) = &request.raw_debug {
  355. raw_debug
  356. .write_http_error(
  357. "anthropic-request-error",
  358. debug_request(request, url, request_body),
  359. HttpDebugResponse {
  360. status: Some(status),
  361. headers: response_headers_for_debug(response_headers),
  362. body: response_body.to_string(),
  363. error_kind: Some("request_error".to_string()),
  364. error: Some(error),
  365. },
  366. )
  367. .await
  368. .context("failed to write Anthropic raw debug request error response")?;
  369. }
  370. Ok(())
  371. }
  372. fn debug_request(request: &ModelRequest, url: &str, body: Value) -> HttpDebugRequest {
  373. HttpDebugRequest {
  374. method: "POST".to_string(),
  375. url: url.to_string(),
  376. headers: request_headers_for_debug(&[
  377. ("x-api-key", request.api_token.clone()),
  378. ("anthropic-version", "2023-06-01".to_string()),
  379. ]),
  380. body,
  381. }
  382. }
  383. fn write_request_error_debug_blocking(
  384. request: &ModelRequest,
  385. url: &str,
  386. request_body: Value,
  387. error: String,
  388. response_kind: &str,
  389. ) {
  390. if let Some(raw_debug) = &request.raw_debug {
  391. let _ = futures::executor::block_on(raw_debug.write_request_error(
  392. response_kind,
  393. debug_request(request, url, request_body),
  394. error,
  395. ));
  396. }
  397. }
  398. #[cfg(test)]
  399. mod tests {
  400. use crate::runner::{ModelRequest, RawDebugConfig, ThinkingConfig};
  401. use reqwest::Client;
  402. use std::path::{Path, PathBuf};
  403. use wiremock::matchers::{body_json, header, method, path};
  404. use wiremock::{Mock, MockServer, ResponseTemplate};
  405. fn read_single_debug_file(root: &Path) -> String {
  406. let mut files = Vec::new();
  407. collect_debug_files(root, &mut files);
  408. assert_eq!(files.len(), 1);
  409. std::fs::read_to_string(&files[0]).expect("read raw debug file")
  410. }
  411. fn collect_debug_files(dir: &Path, files: &mut Vec<PathBuf>) {
  412. for entry in std::fs::read_dir(dir)
  413. .expect("read debug dir")
  414. .collect::<Result<Vec<_>, _>>()
  415. .expect("debug entries")
  416. {
  417. let path = entry.path();
  418. if path.is_dir() {
  419. collect_debug_files(&path, files);
  420. } else {
  421. files.push(path);
  422. }
  423. }
  424. }
  425. #[tokio::test]
  426. async fn extracts_first_text_block() {
  427. let server = MockServer::start().await;
  428. Mock::given(method("POST"))
  429. .and(path("/v1/messages"))
  430. .and(header("x-api-key", "test-token"))
  431. .and(header("anthropic-version", "2023-06-01"))
  432. .and(body_json(serde_json::json!({
  433. "model": "claude-test",
  434. "messages": [{"role": "user", "content": "hello"}],
  435. "temperature": 0.0,
  436. "max_tokens": 1024
  437. })))
  438. .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
  439. "content": [
  440. {"type": "text", "text": "hi from claude"}
  441. ]
  442. })))
  443. .mount(&server)
  444. .await;
  445. let request = ModelRequest {
  446. base_url: server.uri(),
  447. api_token: "test-token".to_string(),
  448. model: "claude-test".to_string(),
  449. prompt: "hello".to_string(),
  450. temperature: 0.0,
  451. max_tokens: 1024,
  452. stream: false,
  453. raw_debug: None,
  454. thinking: None,
  455. };
  456. let response = super::send(&Client::new(), &request)
  457. .await
  458. .expect("response");
  459. assert_eq!(response.status, 200);
  460. assert_eq!(response.text, "hi from claude");
  461. }
  462. #[tokio::test]
  463. async fn sends_thinking_and_omits_temperature_when_enabled() {
  464. let server = MockServer::start().await;
  465. Mock::given(method("POST"))
  466. .and(path("/v1/messages"))
  467. .and(body_json(serde_json::json!({
  468. "model": "claude-test",
  469. "messages": [{"role": "user", "content": "hello"}],
  470. "max_tokens": 1024,
  471. "thinking": {
  472. "type": "enabled",
  473. "budget_tokens": 10000,
  474. "display": "omitted"
  475. }
  476. })))
  477. .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
  478. "content": [
  479. {"type": "text", "text": "hi from claude"}
  480. ]
  481. })))
  482. .mount(&server)
  483. .await;
  484. let mut request = anthropic_request(server.uri());
  485. request.thinking = Some(ThinkingConfig {
  486. enabled: true,
  487. kind: Some("enabled".to_string()),
  488. budget_tokens: Some(10000),
  489. effort: None,
  490. display: Some("omitted".to_string()),
  491. reasoning_effort: None,
  492. reasoning_summary: None,
  493. });
  494. let response = super::send(&Client::new(), &request)
  495. .await
  496. .expect("response");
  497. assert_eq!(response.text, "hi from claude");
  498. }
  499. #[tokio::test]
  500. async fn sends_adaptive_thinking_effort_without_budget_tokens() {
  501. let server = MockServer::start().await;
  502. Mock::given(method("POST"))
  503. .and(path("/v1/messages"))
  504. .and(body_json(serde_json::json!({
  505. "model": "claude-test",
  506. "messages": [{"role": "user", "content": "hello"}],
  507. "max_tokens": 1024,
  508. "thinking": {
  509. "type": "adaptive",
  510. "effort": "high",
  511. "display": "omitted"
  512. }
  513. })))
  514. .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
  515. "content": [
  516. {"type": "text", "text": "hi from claude"}
  517. ]
  518. })))
  519. .mount(&server)
  520. .await;
  521. let mut request = anthropic_request(server.uri());
  522. request.thinking = Some(ThinkingConfig {
  523. enabled: true,
  524. kind: Some("adaptive".to_string()),
  525. budget_tokens: Some(10000),
  526. effort: Some("high".to_string()),
  527. display: Some("omitted".to_string()),
  528. reasoning_effort: None,
  529. reasoning_summary: None,
  530. });
  531. let response = super::send(&Client::new(), &request)
  532. .await
  533. .expect("response");
  534. assert_eq!(response.text, "hi from claude");
  535. }
  536. #[tokio::test]
  537. async fn errors_on_non_success_status() {
  538. let server = MockServer::start().await;
  539. Mock::given(method("POST"))
  540. .and(path("/v1/messages"))
  541. .respond_with(ResponseTemplate::new(401).set_body_json(serde_json::json!({
  542. "error": {
  543. "message": "unauthorized"
  544. }
  545. })))
  546. .mount(&server)
  547. .await;
  548. let request = ModelRequest {
  549. base_url: server.uri(),
  550. api_token: "test-token".to_string(),
  551. model: "claude-test".to_string(),
  552. prompt: "hello".to_string(),
  553. temperature: 0.0,
  554. max_tokens: 1024,
  555. stream: false,
  556. raw_debug: None,
  557. thinking: None,
  558. };
  559. let error = super::send(&Client::new(), &request)
  560. .await
  561. .expect_err("non-success should fail");
  562. assert!(error.to_string().contains("401"));
  563. assert!(error.to_string().contains("unauthorized"));
  564. }
  565. #[tokio::test]
  566. async fn non_success_error_does_not_leak_sensitive_response_body() {
  567. let server = MockServer::start().await;
  568. Mock::given(method("POST"))
  569. .and(path("/v1/messages"))
  570. .respond_with(ResponseTemplate::new(429).set_body_json(serde_json::json!({
  571. "error": {
  572. "message": "rate limited secret prompt sk-leaked-token"
  573. }
  574. })))
  575. .mount(&server)
  576. .await;
  577. let request = ModelRequest {
  578. base_url: server.uri(),
  579. api_token: "test-token".to_string(),
  580. model: "claude-test".to_string(),
  581. prompt: "hello".to_string(),
  582. temperature: 0.0,
  583. max_tokens: 1024,
  584. stream: false,
  585. raw_debug: None,
  586. thinking: None,
  587. };
  588. let message = super::send(&Client::new(), &request)
  589. .await
  590. .expect_err("non-success should fail")
  591. .to_string();
  592. assert!(message.contains("Anthropic"));
  593. assert!(message.contains("429"));
  594. assert!(message.contains("rate limited"));
  595. assert!(!message.contains("secret prompt"));
  596. assert!(!message.contains("sk-leaked-token"));
  597. }
  598. #[tokio::test]
  599. async fn non_success_error_debug_records_request_and_response_without_token() {
  600. let server = MockServer::start().await;
  601. let temp_dir = tempfile::tempdir().expect("create temp dir");
  602. Mock::given(method("POST"))
  603. .and(path("/v1/messages"))
  604. .respond_with(
  605. ResponseTemplate::new(504)
  606. .insert_header("x-request-id", "anthropic-req")
  607. .set_body_json(serde_json::json!({
  608. "error": {
  609. "message": "gateway timeout"
  610. }
  611. })),
  612. )
  613. .mount(&server)
  614. .await;
  615. let request = ModelRequest {
  616. base_url: server.uri(),
  617. api_token: "anthropic-secret-token".to_string(),
  618. model: "claude-test".to_string(),
  619. prompt: "full anthropic prompt".to_string(),
  620. temperature: 0.0,
  621. max_tokens: 1024,
  622. stream: false,
  623. raw_debug: Some(RawDebugConfig::new(
  624. temp_dir.path().to_path_buf(),
  625. "anthropic-claude-test".to_string(),
  626. )),
  627. thinking: None,
  628. };
  629. let _ = super::send(&Client::new(), &request)
  630. .await
  631. .expect_err("non-success should fail");
  632. let raw = read_single_debug_file(temp_dir.path());
  633. let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json");
  634. assert_eq!(debug["request"]["headers"]["x-api-key"], "[REDACTED]");
  635. assert_eq!(
  636. debug["request"]["body"]["messages"][0]["content"],
  637. "full anthropic prompt"
  638. );
  639. assert_eq!(debug["response"]["status"], 504);
  640. assert_eq!(
  641. debug["response"]["headers"]["x-request-id"],
  642. "anthropic-req"
  643. );
  644. assert!(
  645. debug["response"]["body"]
  646. .as_str()
  647. .expect("response body")
  648. .contains("gateway timeout")
  649. );
  650. assert!(!raw.contains("anthropic-secret-token"));
  651. }
  652. #[tokio::test]
  653. async fn request_error_debug_records_request_and_local_error_without_token() {
  654. let temp_dir = tempfile::tempdir().expect("create temp dir");
  655. let request = ModelRequest {
  656. base_url: "http://127.0.0.1:9".to_string(),
  657. api_token: "anthropic-secret-token".to_string(),
  658. model: "claude-test".to_string(),
  659. prompt: "prompt before anthropic connect error".to_string(),
  660. temperature: 0.0,
  661. max_tokens: 1024,
  662. stream: false,
  663. raw_debug: Some(RawDebugConfig::new(
  664. temp_dir.path().to_path_buf(),
  665. "anthropic-claude-test".to_string(),
  666. )),
  667. thinking: None,
  668. };
  669. let _ = super::send(&Client::new(), &request)
  670. .await
  671. .expect_err("connection should fail");
  672. let raw = read_single_debug_file(temp_dir.path());
  673. let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json");
  674. assert_eq!(debug["request"]["method"], "POST");
  675. assert!(
  676. debug["request"]["url"]
  677. .as_str()
  678. .expect("request url")
  679. .ends_with("/v1/messages")
  680. );
  681. assert_eq!(debug["request"]["headers"]["x-api-key"], "[REDACTED]");
  682. assert_eq!(
  683. debug["request"]["body"]["messages"][0]["content"],
  684. "prompt before anthropic connect error"
  685. );
  686. assert_eq!(debug["response"]["status"], serde_json::Value::Null);
  687. assert_eq!(debug["response"]["error_kind"], "request_error");
  688. assert!(
  689. debug["response"]["error"]
  690. .as_str()
  691. .expect("local error")
  692. .contains("failed to send Anthropic messages request")
  693. );
  694. assert!(!raw.contains("anthropic-secret-token"));
  695. }
  696. #[tokio::test]
  697. async fn base_url_with_v1_prefix_does_not_duplicate_messages_path() {
  698. let server = MockServer::start().await;
  699. Mock::given(method("POST"))
  700. .and(path("/v1/messages"))
  701. .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
  702. "content": [
  703. {"type": "text", "text": "prefixed"}
  704. ]
  705. })))
  706. .mount(&server)
  707. .await;
  708. let request = ModelRequest {
  709. base_url: format!("{}/v1", server.uri()),
  710. api_token: "test-token".to_string(),
  711. model: "claude-test".to_string(),
  712. prompt: "hello".to_string(),
  713. temperature: 0.0,
  714. max_tokens: 1024,
  715. stream: false,
  716. raw_debug: None,
  717. thinking: None,
  718. };
  719. let response = super::send(&Client::new(), &request)
  720. .await
  721. .expect("response");
  722. assert_eq!(response.text, "prefixed");
  723. }
  724. #[tokio::test]
  725. async fn extracts_streaming_messages_text_and_ttft() {
  726. let server = MockServer::start().await;
  727. let temp_dir = tempfile::tempdir().expect("create temp dir");
  728. Mock::given(method("POST"))
  729. .and(path("/v1/messages"))
  730. .and(header("x-api-key", "test-token"))
  731. .and(header("anthropic-version", "2023-06-01"))
  732. .and(body_json(serde_json::json!({
  733. "model": "claude-test",
  734. "messages": [{"role": "user", "content": "hello"}],
  735. "temperature": 0.0,
  736. "max_tokens": 1024,
  737. "stream": true
  738. })))
  739. .respond_with(
  740. ResponseTemplate::new(200)
  741. .insert_header("content-type", "text/event-stream")
  742. .set_body_string(
  743. "event: content_block_delta\n\
  744. data: {\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"hi \"}}\n\n\
  745. event: content_block_delta\n\
  746. data: {\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"there\"}}\n\n\
  747. event: message_stop\n\
  748. data: {\"type\":\"message_stop\"}\n\n",
  749. ),
  750. )
  751. .mount(&server)
  752. .await;
  753. let request = ModelRequest {
  754. base_url: server.uri(),
  755. api_token: "test-token".to_string(),
  756. model: "claude-test".to_string(),
  757. prompt: "hello".to_string(),
  758. temperature: 0.0,
  759. max_tokens: 1024,
  760. stream: true,
  761. raw_debug: Some(RawDebugConfig::new(
  762. temp_dir.path().to_path_buf(),
  763. "anthropic-claude-test".to_string(),
  764. )),
  765. thinking: None,
  766. };
  767. let response = super::send_stream(&Client::new(), &request)
  768. .await
  769. .expect("stream response");
  770. assert_eq!(response.status, 200);
  771. assert_eq!(response.text, "hi there");
  772. assert!(response.first_token_ms.is_some());
  773. assert_eq!(response.chunk_elapsed_ms.len(), 1);
  774. let raw = read_single_debug_file(temp_dir.path());
  775. assert!(raw.contains("event: content_block_delta"));
  776. assert!(raw.contains("\"text\":\"hi \""));
  777. }
  778. #[tokio::test]
  779. async fn empty_stream_request_error_debug_records_request_and_raw_stream() {
  780. let server = MockServer::start().await;
  781. let temp_dir = tempfile::tempdir().expect("create temp dir");
  782. Mock::given(method("POST"))
  783. .and(path("/v1/messages"))
  784. .respond_with(
  785. ResponseTemplate::new(200)
  786. .insert_header("content-type", "text/event-stream")
  787. .set_body_string(
  788. "event: message_start\n\
  789. data: {\"type\":\"message_start\"}\n\n\
  790. event: message_stop\n\
  791. data: {\"type\":\"message_stop\"}\n\n",
  792. ),
  793. )
  794. .mount(&server)
  795. .await;
  796. let request = ModelRequest {
  797. base_url: server.uri(),
  798. api_token: "anthropic-secret-token".to_string(),
  799. model: "claude-test".to_string(),
  800. prompt: "prompt before empty stream".to_string(),
  801. temperature: 0.0,
  802. max_tokens: 1024,
  803. stream: true,
  804. raw_debug: Some(
  805. RawDebugConfig::new(
  806. temp_dir.path().to_path_buf(),
  807. "anthropic-claude-test".to_string(),
  808. )
  809. .with_success_raw(false),
  810. ),
  811. thinking: None,
  812. };
  813. let error = super::send_stream(&Client::new(), &request)
  814. .await
  815. .expect_err("empty stream should fail");
  816. assert!(
  817. error
  818. .to_string()
  819. .contains("completed without producing any content")
  820. );
  821. let raw = read_single_debug_file(temp_dir.path());
  822. let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json");
  823. assert_eq!(
  824. debug["request"]["body"]["messages"][0]["content"],
  825. "prompt before empty stream"
  826. );
  827. assert_eq!(debug["response"]["status"], 200);
  828. assert_eq!(debug["response"]["error_kind"], "request_error");
  829. assert!(
  830. debug["response"]["body"]
  831. .as_str()
  832. .expect("response body")
  833. .contains("event: message_stop")
  834. );
  835. assert!(!raw.contains("anthropic-secret-token"));
  836. }
  837. #[tokio::test]
  838. async fn interrupted_stream_debug_records_status_headers_and_partial_stream() {
  839. use tokio::io::{AsyncReadExt, AsyncWriteExt};
  840. use tokio::net::TcpListener;
  841. let listener = TcpListener::bind("127.0.0.1:0")
  842. .await
  843. .expect("bind listener");
  844. let addr = listener.local_addr().expect("listener addr");
  845. let server = tokio::spawn(async move {
  846. let (mut socket, _) = listener.accept().await.expect("accept request");
  847. let mut buffer = [0_u8; 4096];
  848. let _ = socket.read(&mut buffer).await.expect("read request");
  849. let chunk = "event: ping\ndata: {\"type\":\"ping\"}\n\n";
  850. let response = format!(
  851. "HTTP/1.1 200 OK\r\n\
  852. content-type: text/event-stream\r\n\
  853. transfer-encoding: chunked\r\n\
  854. x-request-id: broken-stream\r\n\
  855. \r\n\
  856. {:x}\r\n{}\r\n\
  857. zz\r\nbroken\r\n",
  858. chunk.len(),
  859. chunk
  860. );
  861. socket
  862. .write_all(response.as_bytes())
  863. .await
  864. .expect("write malformed response");
  865. });
  866. let temp_dir = tempfile::tempdir().expect("create temp dir");
  867. let request = ModelRequest {
  868. base_url: format!("http://{addr}"),
  869. api_token: "anthropic-secret-token".to_string(),
  870. model: "claude-test".to_string(),
  871. prompt: "prompt before broken stream".to_string(),
  872. temperature: 0.0,
  873. max_tokens: 1024,
  874. stream: true,
  875. raw_debug: Some(
  876. RawDebugConfig::new(
  877. temp_dir.path().to_path_buf(),
  878. "anthropic-claude-test".to_string(),
  879. )
  880. .with_success_raw(false),
  881. ),
  882. thinking: None,
  883. };
  884. let error = super::send_stream(&Client::new(), &request)
  885. .await
  886. .expect_err("malformed chunked stream should fail");
  887. server.await.expect("server task");
  888. assert!(error.to_string().contains("stream interrupted"));
  889. let raw = read_single_debug_file(temp_dir.path());
  890. let debug: serde_json::Value = serde_json::from_str(&raw).expect("debug json");
  891. assert_eq!(debug["response"]["status"], 200);
  892. assert_eq!(
  893. debug["response"]["headers"]["x-request-id"],
  894. "broken-stream"
  895. );
  896. assert_eq!(debug["response"]["error_kind"], "request_error");
  897. assert!(
  898. debug["response"]["body"]
  899. .as_str()
  900. .expect("partial stream")
  901. .contains("event: ping")
  902. );
  903. assert!(
  904. debug["response"]["error"]
  905. .as_str()
  906. .expect("stream error")
  907. .contains("Anthropic stream interrupted")
  908. );
  909. assert!(!raw.contains("anthropic-secret-token"));
  910. }
  911. fn anthropic_request(base_url: String) -> ModelRequest {
  912. ModelRequest {
  913. base_url,
  914. api_token: "test-token".to_string(),
  915. model: "claude-test".to_string(),
  916. prompt: "hello".to_string(),
  917. temperature: 0.0,
  918. max_tokens: 1024,
  919. stream: false,
  920. raw_debug: None,
  921. thinking: None,
  922. }
  923. }
  924. }