From f17c0af5d704ac9d15d27a3bb8e91f1dfe15c832 Mon Sep 17 00:00:00 2001 From: orangels Date: Sat, 9 May 2026 17:51:54 +0800 Subject: [PATCH] feat: tighten aime final answer parsing --- README.md | 2 ++ docs/USAGE.zh-CN.md | 2 ++ docs/testing-guide.md | 2 ++ src/benchmarks/aime.rs | 5 +++-- src/benchmarks/judge.rs | 36 +++++++++++++++++++++++++++--------- 5 files changed, 36 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index dcb4178..41ec546 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,8 @@ cargo run -- bench aime2026 \ --limit 10 ``` +AIME prompts ask the model to put the final answer in `\boxed{}`. Scoring extracts the last boxed integer first, then falls back only to explicit final-answer forms such as `Final answer: 393`, `Answer: 393`, or `The answer is 393`; unconstrained trailing integers are treated as `no_answer`. + Run a GPQA-Diamond benchmark: ```bash diff --git a/docs/USAGE.zh-CN.md b/docs/USAGE.zh-CN.md index 814bcd6..06174ea 100644 --- a/docs/USAGE.zh-CN.md +++ b/docs/USAGE.zh-CN.md @@ -205,6 +205,8 @@ cargo run -- bench aime2026 \ --concurrency 4 ``` +AIME prompt 会要求模型把最终答案写进 `\boxed{}`。评分时优先提取最后一个 boxed integer,其次只接受 `Final answer: 393`、`Answer: 393`、`The answer is 393` 这类明确 final-answer 格式;解释文本末尾的普通数字不会再被当成答案,会记为 `no_answer`。 + GPQA-Diamond 小样本: ```bash diff --git a/docs/testing-guide.md b/docs/testing-guide.md index b462ba8..5b4952c 100644 --- a/docs/testing-guide.md +++ b/docs/testing-guide.md @@ -178,6 +178,8 @@ cargo run -- bench aime2026 --provider anthropic --model anthropic/claude-sonnet cargo run -- bench aime2026 --provider google --model gemini-3-pro-preview --limit 10 ``` +AIME prompt 会要求模型把最终答案写进 `\boxed{}`。评分时优先提取最后一个 boxed integer,其次只接受 `Final answer: 393`、`Answer: 393`、`The answer is 393` 这类明确 final-answer 格式;解释文本末尾的普通数字不会再被当成答案,会记为 `no_answer`。 + ### 2.3 GPQA Diamond(研究生级别多选题) ```bash diff --git a/src/benchmarks/aime.rs b/src/benchmarks/aime.rs index 0d935fc..b1a539c 100644 --- a/src/benchmarks/aime.rs +++ b/src/benchmarks/aime.rs @@ -74,7 +74,7 @@ pub fn load_cases(data_dir: &Path) -> Result { impl AimeCase { pub fn prompt(&self) -> String { format!( - "Solve the following AIME problem.\n\n{}\n\nRespond with the final integer answer.", + "Solve the following AIME problem. Think step by step before answering.\n\n{}\n\nPut your final answer in \\boxed{{}}.", self.problem ) } @@ -95,7 +95,8 @@ mod tests { let prompt = case.prompt(); assert!(prompt.contains("What is 20 + 22?")); - assert!(prompt.contains("final integer answer")); + assert!(prompt.contains("Think step by step before answering.")); + assert!(prompt.contains(r"\boxed{}")); } #[test] diff --git a/src/benchmarks/judge.rs b/src/benchmarks/judge.rs index 500819f..556a84b 100644 --- a/src/benchmarks/judge.rs +++ b/src/benchmarks/judge.rs @@ -2,15 +2,17 @@ use regex::Regex; pub fn extract_final_integer(text: &str) -> Option { let boxed = Regex::new(r"\\boxed\{\s*(-?\d+)\s*\}").expect("valid boxed integer regex"); - if let Some(captures) = boxed.captures(text) { + if let Some(captures) = boxed.captures_iter(text).last() { return captures.get(1).map(|value| value.as_str().to_string()); } - let integer = Regex::new(r"-?\d+").expect("valid integer regex"); - integer - .find_iter(text) + let explicit = + Regex::new(r"(?i)\b(?:final\s+answer|answer|the\s+answer\s+is)\s*(?:is|:|-)?\s*(-?\d+)\b") + .expect("valid explicit integer answer regex"); + explicit + .captures_iter(text) .last() - .map(|value| value.as_str().to_string()) + .and_then(|captures| captures.get(1).map(|value| value.as_str().to_string())) } pub fn extract_choice(text: &str) -> Option { @@ -42,9 +44,9 @@ mod tests { use super::*; #[test] - fn extracts_final_integer_from_plain_answer() { + fn extracts_final_integer_from_answer_prefix() { assert_eq!( - extract_final_integer("The answer is 42."), + extract_final_integer("Final answer: 42."), Some("42".to_string()) ); } @@ -57,6 +59,22 @@ mod tests { ); } + #[test] + fn extracts_last_boxed_integer() { + assert_eq!( + extract_final_integer(r"First \boxed{12}, final \boxed{34}"), + Some("34".to_string()) + ); + } + + #[test] + fn ignores_trailing_integer_without_explicit_final_answer() { + assert_eq!( + extract_final_integer("The answer should be 393. I verified 30 cases."), + None + ); + } + #[test] fn extracts_choice_from_answer_prefix() { assert_eq!(extract_choice("Answer: C"), Some('C')); @@ -87,8 +105,8 @@ mod tests { #[test] fn judges_integer_by_extracted_value() { - assert!(judge_integer("Final: 42.", "42")); - assert!(!judge_integer("Final: 43.", "42")); + assert!(judge_integer("Final answer: 42.", "42")); + assert!(!judge_integer("Final answer: 43.", "42")); } #[test]