Просмотр исходного кода

feat: align gpqa scoring with simple evals

pull/1/head
orangels 1 месяц назад
Родитель
Сommit
e0aca08f36
6 измененных файлов: 47 добавлений и 47 удалений
  1. +2
    -0
      README.md
  2. +2
    -0
      docs/USAGE.zh-CN.md
  3. +2
    -0
      docs/testing-guide.md
  4. +9
    -7
      src/benchmarks/gpqa.rs
  5. +15
    -38
      src/benchmarks/judge.rs
  6. +17
    -2
      src/cli.rs

+ 2
- 0
README.md Просмотреть файл

@@ -205,6 +205,8 @@ cargo run -- bench gpqa-diamond \

Omit `--limit` to run all locally available cases.

GPQA-Diamond prompts and scoring follow the OpenAI `simple-evals` style: the model is instructed to put `Answer: $LETTER` on the last line, where `LETTER` is one of `A`, `B`, `C`, or `D`. Scoring extracts answers with the same strict `Answer:` pattern, so a bare final `C` or prose such as `I choose C` is treated as `no_answer`.

## RPM Testing

Run a sustained RPM test. This is the default mode and starts requests at a stable interval:


+ 2
- 0
docs/USAGE.zh-CN.md Просмотреть файл

@@ -224,6 +224,8 @@ cargo run -- bench gpqa-diamond \
--concurrency 4
```

GPQA-Diamond 的 prompt 和评分按 OpenAI `simple-evals` 风格处理:要求模型最后一行输出 `Answer: $LETTER`,其中 `LETTER` 是 `A`、`B`、`C` 或 `D`。评分只按这个严格 `Answer:` 格式提取答案;如果模型只在最后单独输出 `C`,或输出 `I choose C`,会记为 `no_answer`。

benchmark 会输出:

- accuracy


+ 2
- 0
docs/testing-guide.md Просмотреть файл

@@ -194,6 +194,8 @@ cargo run -- bench gpqa-diamond --provider anthropic --model anthropic/claude-so
cargo run -- bench gpqa-diamond --provider google --model gemini-3-pro-preview --limit 10
```

GPQA-Diamond 的 prompt 和评分按 OpenAI `simple-evals` 风格处理:要求模型最后一行输出 `Answer: $LETTER`,其中 `LETTER` 是 `A`、`B`、`C` 或 `D`。评分只按这个严格 `Answer:` 格式提取答案;如果模型只在最后单独输出 `C`,或输出 `I choose C`,会记为 `no_answer`。

### 2.4 精度参考基线

| 数据集 | claude-haiku-4.5 | claude-sonnet-4 | gpt-4o-mini |


+ 9
- 7
src/benchmarks/gpqa.rs Просмотреть файл

@@ -160,12 +160,12 @@ impl GpqaCase {
let choices = self
.choices
.iter()
.map(|(label, text)| format!("{label}. {text}"))
.map(|(label, text)| format!("{label}) {text}"))
.collect::<Vec<_>>()
.join("\n");

format!(
"Answer the following multiple-choice question.\n\n{}\n\n{}\n\nPlease answer with exactly one letter: A, B, C, or D.",
"Answer the following multiple choice question.\nThe last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n{}\n\n{}",
self.question, choices
)
}
@@ -192,11 +192,13 @@ mod tests {
let prompt = case.prompt();

assert!(prompt.contains("Which option is correct?"));
assert!(prompt.contains("A. Alpha"));
assert!(prompt.contains("B. Beta"));
assert!(prompt.contains("C. Gamma"));
assert!(prompt.contains("D. Delta"));
assert!(prompt.contains("answer with exactly one letter"));
assert!(prompt.contains("A) Alpha"));
assert!(prompt.contains("B) Beta"));
assert!(prompt.contains("C) Gamma"));
assert!(prompt.contains("D) Delta"));
assert!(prompt.contains("The last line of your response"));
assert!(prompt.contains("'Answer: $LETTER'"));
assert!(prompt.contains("Think step by step before answering."));
}

#[test]


+ 15
- 38
src/benchmarks/judge.rs Просмотреть файл

@@ -14,19 +14,12 @@ pub fn extract_final_integer(text: &str) -> Option<String> {
}

pub fn extract_choice(text: &str) -> Option<char> {
let explicit = Regex::new(
r"(?i)\b(?:final\s+answer|answer|choose|choice)\s*(?:is|:|-)?\s*\(?([A-D])\)?\b",
)
.expect("valid explicit choice regex");
if let Some(captures) = explicit.captures_iter(text).last() {
return choice_from_capture(&captures, 1);
}

let whole_output =
Regex::new(r"(?i)^(?:([A-D])|\(([A-D])\))$").expect("valid whole output choice regex");
whole_output.captures(text.trim()).and_then(|captures| {
choice_from_capture(&captures, 1).or_else(|| choice_from_capture(&captures, 2))
})
let explicit =
Regex::new(r"(?i)Answer[ \t]*:[ \t]*\$?([A-D])\$?").expect("valid GPQA answer regex");
explicit
.captures_iter(text)
.last()
.and_then(|captures| choice_from_capture(&captures, 1))
}

fn choice_from_capture(captures: &regex::Captures<'_>, index: usize) -> Option<char> {
@@ -70,42 +63,26 @@ mod tests {
}

#[test]
fn extracts_choice_from_parenthesized_lowercase() {
assert_eq!(extract_choice("I choose (b)."), Some('B'));
fn extracts_choice_from_dollar_wrapped_answer() {
assert_eq!(extract_choice("Answer: $B$"), Some('B'));
}

#[test]
fn extracts_choice_from_final_answer_instead_of_earlier_prose_option() {
fn extracts_last_answer_prefix() {
assert_eq!(
extract_choice(
"Option A is tempting, but after checking the calculation the answer is D"
),
extract_choice("Answer: A was my first thought.\nAnswer: D"),
Some('D')
);
}

#[test]
fn extracts_choice_from_final_answer_after_considered_options() {
assert_eq!(
extract_choice("I considered A and B before deciding. Final answer: C"),
Some('C')
);
fn ignores_choice_without_answer_prefix() {
assert_eq!(extract_choice("I choose (b)."), None);
}

#[test]
fn extracts_last_explicit_choice() {
assert_eq!(
extract_choice("Answer: A was my first thought. Final answer: D"),
Some('D')
);
}

#[test]
fn ignores_prose_without_final_choice() {
assert_eq!(
extract_choice("This explanation has no final option."),
None
);
fn ignores_bare_final_line_choice() {
assert_eq!(extract_choice("<think>...</think>\n\nC"), None);
}

#[test]
@@ -116,7 +93,7 @@ mod tests {

#[test]
fn judges_choice_by_extracted_value() {
assert!(judge_choice("I choose (b).", 'B'));
assert!(judge_choice("Answer: B", 'B'));
assert!(!judge_choice("Answer: C", 'B'));
}
}

+ 17
- 2
src/cli.rs Просмотреть файл

@@ -138,7 +138,7 @@ pub enum BenchCommand {
limit: Option<usize>,
#[arg(long, num_args = 0..=1, default_missing_value = "true")]
stream: Option<bool>,
#[arg(long, default_value_t = 32768)]
#[arg(long, default_value_t = 20_000)]
max_tokens: u32,
#[arg(long)]
debug_raw: bool,
@@ -170,7 +170,7 @@ pub enum BenchCommand {
limit: Option<usize>,
#[arg(long, num_args = 0..=1, default_missing_value = "true")]
stream: Option<bool>,
#[arg(long, default_value_t = 32768)]
#[arg(long, default_value_t = 20_000)]
max_tokens: u32,
#[arg(long)]
debug_raw: bool,
@@ -1575,6 +1575,21 @@ mod tests {
assert_eq!(stream, Some(false));
}

#[test]
fn bench_command_defaults_max_tokens_to_20000() {
let cli = Cli::try_parse_from(["lq_token_test", "bench", "gpqa-diamond"])
.expect("parse gpqa bench");

let Command::Bench {
command: BenchCommand::GpqaDiamond { max_tokens, .. },
} = cli.command
else {
panic!("expected gpqa-diamond bench command");
};

assert_eq!(max_tokens, 20_000);
}

#[test]
fn check_command_parses_debug_raw() {
let cli = Cli::try_parse_from([


Загрузка…
Отмена
Сохранить