| @@ -205,6 +205,8 @@ cargo run -- bench gpqa-diamond \ | |||||
| Omit `--limit` to run all locally available cases. | Omit `--limit` to run all locally available cases. | ||||
| GPQA-Diamond prompts and scoring follow the OpenAI `simple-evals` style: the model is instructed to put `Answer: $LETTER` on the last line, where `LETTER` is one of `A`, `B`, `C`, or `D`. Scoring extracts answers with the same strict `Answer:` pattern, so a bare final `C` or prose such as `I choose C` is treated as `no_answer`. | |||||
| ## RPM Testing | ## RPM Testing | ||||
| Run a sustained RPM test. This is the default mode and starts requests at a stable interval: | Run a sustained RPM test. This is the default mode and starts requests at a stable interval: | ||||
| @@ -224,6 +224,8 @@ cargo run -- bench gpqa-diamond \ | |||||
| --concurrency 4 | --concurrency 4 | ||||
| ``` | ``` | ||||
| GPQA-Diamond 的 prompt 和评分按 OpenAI `simple-evals` 风格处理:要求模型最后一行输出 `Answer: $LETTER`,其中 `LETTER` 是 `A`、`B`、`C` 或 `D`。评分只按这个严格 `Answer:` 格式提取答案;如果模型只在最后单独输出 `C`,或输出 `I choose C`,会记为 `no_answer`。 | |||||
| benchmark 会输出: | benchmark 会输出: | ||||
| - accuracy | - accuracy | ||||
| @@ -194,6 +194,8 @@ cargo run -- bench gpqa-diamond --provider anthropic --model anthropic/claude-so | |||||
| cargo run -- bench gpqa-diamond --provider google --model gemini-3-pro-preview --limit 10 | cargo run -- bench gpqa-diamond --provider google --model gemini-3-pro-preview --limit 10 | ||||
| ``` | ``` | ||||
| GPQA-Diamond 的 prompt 和评分按 OpenAI `simple-evals` 风格处理:要求模型最后一行输出 `Answer: $LETTER`,其中 `LETTER` 是 `A`、`B`、`C` 或 `D`。评分只按这个严格 `Answer:` 格式提取答案;如果模型只在最后单独输出 `C`,或输出 `I choose C`,会记为 `no_answer`。 | |||||
| ### 2.4 精度参考基线 | ### 2.4 精度参考基线 | ||||
| | 数据集 | claude-haiku-4.5 | claude-sonnet-4 | gpt-4o-mini | | | 数据集 | claude-haiku-4.5 | claude-sonnet-4 | gpt-4o-mini | | ||||
| @@ -160,12 +160,12 @@ impl GpqaCase { | |||||
| let choices = self | let choices = self | ||||
| .choices | .choices | ||||
| .iter() | .iter() | ||||
| .map(|(label, text)| format!("{label}. {text}")) | |||||
| .map(|(label, text)| format!("{label}) {text}")) | |||||
| .collect::<Vec<_>>() | .collect::<Vec<_>>() | ||||
| .join("\n"); | .join("\n"); | ||||
| format!( | format!( | ||||
| "Answer the following multiple-choice question.\n\n{}\n\n{}\n\nPlease answer with exactly one letter: A, B, C, or D.", | |||||
| "Answer the following multiple choice question.\nThe last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n{}\n\n{}", | |||||
| self.question, choices | self.question, choices | ||||
| ) | ) | ||||
| } | } | ||||
| @@ -192,11 +192,13 @@ mod tests { | |||||
| let prompt = case.prompt(); | let prompt = case.prompt(); | ||||
| assert!(prompt.contains("Which option is correct?")); | assert!(prompt.contains("Which option is correct?")); | ||||
| assert!(prompt.contains("A. Alpha")); | |||||
| assert!(prompt.contains("B. Beta")); | |||||
| assert!(prompt.contains("C. Gamma")); | |||||
| assert!(prompt.contains("D. Delta")); | |||||
| assert!(prompt.contains("answer with exactly one letter")); | |||||
| assert!(prompt.contains("A) Alpha")); | |||||
| assert!(prompt.contains("B) Beta")); | |||||
| assert!(prompt.contains("C) Gamma")); | |||||
| assert!(prompt.contains("D) Delta")); | |||||
| assert!(prompt.contains("The last line of your response")); | |||||
| assert!(prompt.contains("'Answer: $LETTER'")); | |||||
| assert!(prompt.contains("Think step by step before answering.")); | |||||
| } | } | ||||
| #[test] | #[test] | ||||
| @@ -14,19 +14,12 @@ pub fn extract_final_integer(text: &str) -> Option<String> { | |||||
| } | } | ||||
| pub fn extract_choice(text: &str) -> Option<char> { | pub fn extract_choice(text: &str) -> Option<char> { | ||||
| let explicit = Regex::new( | |||||
| r"(?i)\b(?:final\s+answer|answer|choose|choice)\s*(?:is|:|-)?\s*\(?([A-D])\)?\b", | |||||
| ) | |||||
| .expect("valid explicit choice regex"); | |||||
| if let Some(captures) = explicit.captures_iter(text).last() { | |||||
| return choice_from_capture(&captures, 1); | |||||
| } | |||||
| let whole_output = | |||||
| Regex::new(r"(?i)^(?:([A-D])|\(([A-D])\))$").expect("valid whole output choice regex"); | |||||
| whole_output.captures(text.trim()).and_then(|captures| { | |||||
| choice_from_capture(&captures, 1).or_else(|| choice_from_capture(&captures, 2)) | |||||
| }) | |||||
| let explicit = | |||||
| Regex::new(r"(?i)Answer[ \t]*:[ \t]*\$?([A-D])\$?").expect("valid GPQA answer regex"); | |||||
| explicit | |||||
| .captures_iter(text) | |||||
| .last() | |||||
| .and_then(|captures| choice_from_capture(&captures, 1)) | |||||
| } | } | ||||
| fn choice_from_capture(captures: ®ex::Captures<'_>, index: usize) -> Option<char> { | fn choice_from_capture(captures: ®ex::Captures<'_>, index: usize) -> Option<char> { | ||||
| @@ -70,42 +63,26 @@ mod tests { | |||||
| } | } | ||||
| #[test] | #[test] | ||||
| fn extracts_choice_from_parenthesized_lowercase() { | |||||
| assert_eq!(extract_choice("I choose (b)."), Some('B')); | |||||
| fn extracts_choice_from_dollar_wrapped_answer() { | |||||
| assert_eq!(extract_choice("Answer: $B$"), Some('B')); | |||||
| } | } | ||||
| #[test] | #[test] | ||||
| fn extracts_choice_from_final_answer_instead_of_earlier_prose_option() { | |||||
| fn extracts_last_answer_prefix() { | |||||
| assert_eq!( | assert_eq!( | ||||
| extract_choice( | |||||
| "Option A is tempting, but after checking the calculation the answer is D" | |||||
| ), | |||||
| extract_choice("Answer: A was my first thought.\nAnswer: D"), | |||||
| Some('D') | Some('D') | ||||
| ); | ); | ||||
| } | } | ||||
| #[test] | #[test] | ||||
| fn extracts_choice_from_final_answer_after_considered_options() { | |||||
| assert_eq!( | |||||
| extract_choice("I considered A and B before deciding. Final answer: C"), | |||||
| Some('C') | |||||
| ); | |||||
| fn ignores_choice_without_answer_prefix() { | |||||
| assert_eq!(extract_choice("I choose (b)."), None); | |||||
| } | } | ||||
| #[test] | #[test] | ||||
| fn extracts_last_explicit_choice() { | |||||
| assert_eq!( | |||||
| extract_choice("Answer: A was my first thought. Final answer: D"), | |||||
| Some('D') | |||||
| ); | |||||
| } | |||||
| #[test] | |||||
| fn ignores_prose_without_final_choice() { | |||||
| assert_eq!( | |||||
| extract_choice("This explanation has no final option."), | |||||
| None | |||||
| ); | |||||
| fn ignores_bare_final_line_choice() { | |||||
| assert_eq!(extract_choice("<think>...</think>\n\nC"), None); | |||||
| } | } | ||||
| #[test] | #[test] | ||||
| @@ -116,7 +93,7 @@ mod tests { | |||||
| #[test] | #[test] | ||||
| fn judges_choice_by_extracted_value() { | fn judges_choice_by_extracted_value() { | ||||
| assert!(judge_choice("I choose (b).", 'B')); | |||||
| assert!(judge_choice("Answer: B", 'B')); | |||||
| assert!(!judge_choice("Answer: C", 'B')); | assert!(!judge_choice("Answer: C", 'B')); | ||||
| } | } | ||||
| } | } | ||||
| @@ -138,7 +138,7 @@ pub enum BenchCommand { | |||||
| limit: Option<usize>, | limit: Option<usize>, | ||||
| #[arg(long, num_args = 0..=1, default_missing_value = "true")] | #[arg(long, num_args = 0..=1, default_missing_value = "true")] | ||||
| stream: Option<bool>, | stream: Option<bool>, | ||||
| #[arg(long, default_value_t = 32768)] | |||||
| #[arg(long, default_value_t = 20_000)] | |||||
| max_tokens: u32, | max_tokens: u32, | ||||
| #[arg(long)] | #[arg(long)] | ||||
| debug_raw: bool, | debug_raw: bool, | ||||
| @@ -170,7 +170,7 @@ pub enum BenchCommand { | |||||
| limit: Option<usize>, | limit: Option<usize>, | ||||
| #[arg(long, num_args = 0..=1, default_missing_value = "true")] | #[arg(long, num_args = 0..=1, default_missing_value = "true")] | ||||
| stream: Option<bool>, | stream: Option<bool>, | ||||
| #[arg(long, default_value_t = 32768)] | |||||
| #[arg(long, default_value_t = 20_000)] | |||||
| max_tokens: u32, | max_tokens: u32, | ||||
| #[arg(long)] | #[arg(long)] | ||||
| debug_raw: bool, | debug_raw: bool, | ||||
| @@ -1575,6 +1575,21 @@ mod tests { | |||||
| assert_eq!(stream, Some(false)); | assert_eq!(stream, Some(false)); | ||||
| } | } | ||||
| #[test] | |||||
| fn bench_command_defaults_max_tokens_to_20000() { | |||||
| let cli = Cli::try_parse_from(["lq_token_test", "bench", "gpqa-diamond"]) | |||||
| .expect("parse gpqa bench"); | |||||
| let Command::Bench { | |||||
| command: BenchCommand::GpqaDiamond { max_tokens, .. }, | |||||
| } = cli.command | |||||
| else { | |||||
| panic!("expected gpqa-diamond bench command"); | |||||
| }; | |||||
| assert_eq!(max_tokens, 20_000); | |||||
| } | |||||
| #[test] | #[test] | ||||
| fn check_command_parses_debug_raw() { | fn check_command_parses_debug_raw() { | ||||
| let cli = Cli::try_parse_from([ | let cli = Cli::try_parse_from([ | ||||