From e0aca08f36486c5d5211cab6d13f4d6eb7a17ea8 Mon Sep 17 00:00:00 2001 From: orangels Date: Sat, 9 May 2026 17:37:09 +0800 Subject: [PATCH] feat: align gpqa scoring with simple evals --- README.md | 2 ++ docs/USAGE.zh-CN.md | 2 ++ docs/testing-guide.md | 2 ++ src/benchmarks/gpqa.rs | 16 +++++++------ src/benchmarks/judge.rs | 53 ++++++++++++----------------------------- src/cli.rs | 19 +++++++++++++-- 6 files changed, 47 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 76e1dfa..dcb4178 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,8 @@ cargo run -- bench gpqa-diamond \ Omit `--limit` to run all locally available cases. +GPQA-Diamond prompts and scoring follow the OpenAI `simple-evals` style: the model is instructed to put `Answer: $LETTER` on the last line, where `LETTER` is one of `A`, `B`, `C`, or `D`. Scoring extracts answers with the same strict `Answer:` pattern, so a bare final `C` or prose such as `I choose C` is treated as `no_answer`. + ## RPM Testing Run a sustained RPM test. This is the default mode and starts requests at a stable interval: diff --git a/docs/USAGE.zh-CN.md b/docs/USAGE.zh-CN.md index d13bbf1..814bcd6 100644 --- a/docs/USAGE.zh-CN.md +++ b/docs/USAGE.zh-CN.md @@ -224,6 +224,8 @@ cargo run -- bench gpqa-diamond \ --concurrency 4 ``` +GPQA-Diamond 的 prompt 和评分按 OpenAI `simple-evals` 风格处理:要求模型最后一行输出 `Answer: $LETTER`,其中 `LETTER` 是 `A`、`B`、`C` 或 `D`。评分只按这个严格 `Answer:` 格式提取答案;如果模型只在最后单独输出 `C`,或输出 `I choose C`,会记为 `no_answer`。 + benchmark 会输出: - accuracy diff --git a/docs/testing-guide.md b/docs/testing-guide.md index 669134a..b462ba8 100644 --- a/docs/testing-guide.md +++ b/docs/testing-guide.md @@ -194,6 +194,8 @@ cargo run -- bench gpqa-diamond --provider anthropic --model anthropic/claude-so cargo run -- bench gpqa-diamond --provider google --model gemini-3-pro-preview --limit 10 ``` +GPQA-Diamond 的 prompt 和评分按 OpenAI `simple-evals` 风格处理:要求模型最后一行输出 `Answer: $LETTER`,其中 `LETTER` 是 `A`、`B`、`C` 或 `D`。评分只按这个严格 `Answer:` 格式提取答案;如果模型只在最后单独输出 `C`,或输出 `I choose C`,会记为 `no_answer`。 + ### 2.4 精度参考基线 | 数据集 | claude-haiku-4.5 | claude-sonnet-4 | gpt-4o-mini | diff --git a/src/benchmarks/gpqa.rs b/src/benchmarks/gpqa.rs index 294f199..2aedf2d 100644 --- a/src/benchmarks/gpqa.rs +++ b/src/benchmarks/gpqa.rs @@ -160,12 +160,12 @@ impl GpqaCase { let choices = self .choices .iter() - .map(|(label, text)| format!("{label}. {text}")) + .map(|(label, text)| format!("{label}) {text}")) .collect::>() .join("\n"); format!( - "Answer the following multiple-choice question.\n\n{}\n\n{}\n\nPlease answer with exactly one letter: A, B, C, or D.", + "Answer the following multiple choice question.\nThe last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n{}\n\n{}", self.question, choices ) } @@ -192,11 +192,13 @@ mod tests { let prompt = case.prompt(); assert!(prompt.contains("Which option is correct?")); - assert!(prompt.contains("A. Alpha")); - assert!(prompt.contains("B. Beta")); - assert!(prompt.contains("C. Gamma")); - assert!(prompt.contains("D. Delta")); - assert!(prompt.contains("answer with exactly one letter")); + assert!(prompt.contains("A) Alpha")); + assert!(prompt.contains("B) Beta")); + assert!(prompt.contains("C) Gamma")); + assert!(prompt.contains("D) Delta")); + assert!(prompt.contains("The last line of your response")); + assert!(prompt.contains("'Answer: $LETTER'")); + assert!(prompt.contains("Think step by step before answering.")); } #[test] diff --git a/src/benchmarks/judge.rs b/src/benchmarks/judge.rs index a079fad..500819f 100644 --- a/src/benchmarks/judge.rs +++ b/src/benchmarks/judge.rs @@ -14,19 +14,12 @@ pub fn extract_final_integer(text: &str) -> Option { } pub fn extract_choice(text: &str) -> Option { - let explicit = Regex::new( - r"(?i)\b(?:final\s+answer|answer|choose|choice)\s*(?:is|:|-)?\s*\(?([A-D])\)?\b", - ) - .expect("valid explicit choice regex"); - if let Some(captures) = explicit.captures_iter(text).last() { - return choice_from_capture(&captures, 1); - } - - let whole_output = - Regex::new(r"(?i)^(?:([A-D])|\(([A-D])\))$").expect("valid whole output choice regex"); - whole_output.captures(text.trim()).and_then(|captures| { - choice_from_capture(&captures, 1).or_else(|| choice_from_capture(&captures, 2)) - }) + let explicit = + Regex::new(r"(?i)Answer[ \t]*:[ \t]*\$?([A-D])\$?").expect("valid GPQA answer regex"); + explicit + .captures_iter(text) + .last() + .and_then(|captures| choice_from_capture(&captures, 1)) } fn choice_from_capture(captures: ®ex::Captures<'_>, index: usize) -> Option { @@ -70,42 +63,26 @@ mod tests { } #[test] - fn extracts_choice_from_parenthesized_lowercase() { - assert_eq!(extract_choice("I choose (b)."), Some('B')); + fn extracts_choice_from_dollar_wrapped_answer() { + assert_eq!(extract_choice("Answer: $B$"), Some('B')); } #[test] - fn extracts_choice_from_final_answer_instead_of_earlier_prose_option() { + fn extracts_last_answer_prefix() { assert_eq!( - extract_choice( - "Option A is tempting, but after checking the calculation the answer is D" - ), + extract_choice("Answer: A was my first thought.\nAnswer: D"), Some('D') ); } #[test] - fn extracts_choice_from_final_answer_after_considered_options() { - assert_eq!( - extract_choice("I considered A and B before deciding. Final answer: C"), - Some('C') - ); + fn ignores_choice_without_answer_prefix() { + assert_eq!(extract_choice("I choose (b)."), None); } #[test] - fn extracts_last_explicit_choice() { - assert_eq!( - extract_choice("Answer: A was my first thought. Final answer: D"), - Some('D') - ); - } - - #[test] - fn ignores_prose_without_final_choice() { - assert_eq!( - extract_choice("This explanation has no final option."), - None - ); + fn ignores_bare_final_line_choice() { + assert_eq!(extract_choice("...\n\nC"), None); } #[test] @@ -116,7 +93,7 @@ mod tests { #[test] fn judges_choice_by_extracted_value() { - assert!(judge_choice("I choose (b).", 'B')); + assert!(judge_choice("Answer: B", 'B')); assert!(!judge_choice("Answer: C", 'B')); } } diff --git a/src/cli.rs b/src/cli.rs index 666eba8..54a490c 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -138,7 +138,7 @@ pub enum BenchCommand { limit: Option, #[arg(long, num_args = 0..=1, default_missing_value = "true")] stream: Option, - #[arg(long, default_value_t = 32768)] + #[arg(long, default_value_t = 20_000)] max_tokens: u32, #[arg(long)] debug_raw: bool, @@ -170,7 +170,7 @@ pub enum BenchCommand { limit: Option, #[arg(long, num_args = 0..=1, default_missing_value = "true")] stream: Option, - #[arg(long, default_value_t = 32768)] + #[arg(long, default_value_t = 20_000)] max_tokens: u32, #[arg(long)] debug_raw: bool, @@ -1575,6 +1575,21 @@ mod tests { assert_eq!(stream, Some(false)); } + #[test] + fn bench_command_defaults_max_tokens_to_20000() { + let cli = Cli::try_parse_from(["lq_token_test", "bench", "gpqa-diamond"]) + .expect("parse gpqa bench"); + + let Command::Bench { + command: BenchCommand::GpqaDiamond { max_tokens, .. }, + } = cli.command + else { + panic!("expected gpqa-diamond bench command"); + }; + + assert_eq!(max_tokens, 20_000); + } + #[test] fn check_command_parses_debug_raw() { let cli = Cli::try_parse_from([