api模型检测
Non puoi selezionare più di 25 argomenti Gli argomenti devono iniziare con una lettera o un numero, possono includere trattini ('-') e possono essere lunghi fino a 35 caratteri.

118 righe
3.3 KiB

  1. use regex::Regex;
  2. pub fn extract_final_integer(text: &str) -> Option<String> {
  3. let boxed = Regex::new(r"\\boxed\{\s*(-?\d+)\s*\}").expect("valid boxed integer regex");
  4. if let Some(captures) = boxed.captures_iter(text).last() {
  5. return captures.get(1).map(|value| value.as_str().to_string());
  6. }
  7. let explicit =
  8. Regex::new(r"(?i)\b(?:final\s+answer|answer|the\s+answer\s+is)\s*(?:is|:|-)?\s*(-?\d+)\b")
  9. .expect("valid explicit integer answer regex");
  10. explicit
  11. .captures_iter(text)
  12. .last()
  13. .and_then(|captures| captures.get(1).map(|value| value.as_str().to_string()))
  14. }
  15. pub fn extract_choice(text: &str) -> Option<char> {
  16. let explicit =
  17. Regex::new(r"(?i)Answer[ \t]*:[ \t]*\$?([A-D])\$?").expect("valid GPQA answer regex");
  18. explicit
  19. .captures_iter(text)
  20. .last()
  21. .and_then(|captures| choice_from_capture(&captures, 1))
  22. }
  23. fn choice_from_capture(captures: &regex::Captures<'_>, index: usize) -> Option<char> {
  24. captures
  25. .get(index)
  26. .and_then(|value| value.as_str().chars().next())
  27. .map(|choice| choice.to_ascii_uppercase())
  28. }
  29. pub fn judge_integer(output: &str, expected: &str) -> bool {
  30. extract_final_integer(output).is_some_and(|actual| actual == expected.trim())
  31. }
  32. pub fn judge_choice(output: &str, expected: char) -> bool {
  33. extract_choice(output).is_some_and(|actual| actual == expected.to_ascii_uppercase())
  34. }
  35. #[cfg(test)]
  36. mod tests {
  37. use super::*;
  38. #[test]
  39. fn extracts_final_integer_from_answer_prefix() {
  40. assert_eq!(
  41. extract_final_integer("Final answer: 42."),
  42. Some("42".to_string())
  43. );
  44. }
  45. #[test]
  46. fn extracts_final_integer_from_boxed_answer() {
  47. assert_eq!(
  48. extract_final_integer(r"Final: \boxed{17}"),
  49. Some("17".to_string())
  50. );
  51. }
  52. #[test]
  53. fn extracts_last_boxed_integer() {
  54. assert_eq!(
  55. extract_final_integer(r"First \boxed{12}, final \boxed{34}"),
  56. Some("34".to_string())
  57. );
  58. }
  59. #[test]
  60. fn ignores_trailing_integer_without_explicit_final_answer() {
  61. assert_eq!(
  62. extract_final_integer("The answer should be 393. I verified 30 cases."),
  63. None
  64. );
  65. }
  66. #[test]
  67. fn extracts_choice_from_answer_prefix() {
  68. assert_eq!(extract_choice("Answer: C"), Some('C'));
  69. }
  70. #[test]
  71. fn extracts_choice_from_dollar_wrapped_answer() {
  72. assert_eq!(extract_choice("Answer: $B$"), Some('B'));
  73. }
  74. #[test]
  75. fn extracts_last_answer_prefix() {
  76. assert_eq!(
  77. extract_choice("Answer: A was my first thought.\nAnswer: D"),
  78. Some('D')
  79. );
  80. }
  81. #[test]
  82. fn ignores_choice_without_answer_prefix() {
  83. assert_eq!(extract_choice("I choose (b)."), None);
  84. }
  85. #[test]
  86. fn ignores_bare_final_line_choice() {
  87. assert_eq!(extract_choice("<think>...</think>\n\nC"), None);
  88. }
  89. #[test]
  90. fn judges_integer_by_extracted_value() {
  91. assert!(judge_integer("Final answer: 42.", "42"));
  92. assert!(!judge_integer("Final answer: 43.", "42"));
  93. }
  94. #[test]
  95. fn judges_choice_by_extracted_value() {
  96. assert!(judge_choice("Answer: B", 'B'));
  97. assert!(!judge_choice("Answer: C", 'B'));
  98. }
  99. }