| @@ -205,6 +205,8 @@ cargo run -- bench gpqa-diamond \ | |||
| Omit `--limit` to run all locally available cases. | |||
| GPQA-Diamond prompts and scoring follow the OpenAI `simple-evals` style: the model is instructed to put `Answer: $LETTER` on the last line, where `LETTER` is one of `A`, `B`, `C`, or `D`. Scoring extracts answers with the same strict `Answer:` pattern, so a bare final `C` or prose such as `I choose C` is treated as `no_answer`. | |||
| ## RPM Testing | |||
| Run a sustained RPM test. This is the default mode and starts requests at a stable interval: | |||
| @@ -224,6 +224,8 @@ cargo run -- bench gpqa-diamond \ | |||
| --concurrency 4 | |||
| ``` | |||
| GPQA-Diamond 的 prompt 和评分按 OpenAI `simple-evals` 风格处理:要求模型最后一行输出 `Answer: $LETTER`,其中 `LETTER` 是 `A`、`B`、`C` 或 `D`。评分只按这个严格 `Answer:` 格式提取答案;如果模型只在最后单独输出 `C`,或输出 `I choose C`,会记为 `no_answer`。 | |||
| benchmark 会输出: | |||
| - accuracy | |||
| @@ -194,6 +194,8 @@ cargo run -- bench gpqa-diamond --provider anthropic --model anthropic/claude-so | |||
| cargo run -- bench gpqa-diamond --provider google --model gemini-3-pro-preview --limit 10 | |||
| ``` | |||
| GPQA-Diamond 的 prompt 和评分按 OpenAI `simple-evals` 风格处理:要求模型最后一行输出 `Answer: $LETTER`,其中 `LETTER` 是 `A`、`B`、`C` 或 `D`。评分只按这个严格 `Answer:` 格式提取答案;如果模型只在最后单独输出 `C`,或输出 `I choose C`,会记为 `no_answer`。 | |||
| ### 2.4 精度参考基线 | |||
| | 数据集 | claude-haiku-4.5 | claude-sonnet-4 | gpt-4o-mini | | |||
| @@ -160,12 +160,12 @@ impl GpqaCase { | |||
| let choices = self | |||
| .choices | |||
| .iter() | |||
| .map(|(label, text)| format!("{label}. {text}")) | |||
| .map(|(label, text)| format!("{label}) {text}")) | |||
| .collect::<Vec<_>>() | |||
| .join("\n"); | |||
| format!( | |||
| "Answer the following multiple-choice question.\n\n{}\n\n{}\n\nPlease answer with exactly one letter: A, B, C, or D.", | |||
| "Answer the following multiple choice question.\nThe last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n{}\n\n{}", | |||
| self.question, choices | |||
| ) | |||
| } | |||
| @@ -192,11 +192,13 @@ mod tests { | |||
| let prompt = case.prompt(); | |||
| assert!(prompt.contains("Which option is correct?")); | |||
| assert!(prompt.contains("A. Alpha")); | |||
| assert!(prompt.contains("B. Beta")); | |||
| assert!(prompt.contains("C. Gamma")); | |||
| assert!(prompt.contains("D. Delta")); | |||
| assert!(prompt.contains("answer with exactly one letter")); | |||
| assert!(prompt.contains("A) Alpha")); | |||
| assert!(prompt.contains("B) Beta")); | |||
| assert!(prompt.contains("C) Gamma")); | |||
| assert!(prompt.contains("D) Delta")); | |||
| assert!(prompt.contains("The last line of your response")); | |||
| assert!(prompt.contains("'Answer: $LETTER'")); | |||
| assert!(prompt.contains("Think step by step before answering.")); | |||
| } | |||
| #[test] | |||
| @@ -14,19 +14,12 @@ pub fn extract_final_integer(text: &str) -> Option<String> { | |||
| } | |||
| pub fn extract_choice(text: &str) -> Option<char> { | |||
| let explicit = Regex::new( | |||
| r"(?i)\b(?:final\s+answer|answer|choose|choice)\s*(?:is|:|-)?\s*\(?([A-D])\)?\b", | |||
| ) | |||
| .expect("valid explicit choice regex"); | |||
| if let Some(captures) = explicit.captures_iter(text).last() { | |||
| return choice_from_capture(&captures, 1); | |||
| } | |||
| let whole_output = | |||
| Regex::new(r"(?i)^(?:([A-D])|\(([A-D])\))$").expect("valid whole output choice regex"); | |||
| whole_output.captures(text.trim()).and_then(|captures| { | |||
| choice_from_capture(&captures, 1).or_else(|| choice_from_capture(&captures, 2)) | |||
| }) | |||
| let explicit = | |||
| Regex::new(r"(?i)Answer[ \t]*:[ \t]*\$?([A-D])\$?").expect("valid GPQA answer regex"); | |||
| explicit | |||
| .captures_iter(text) | |||
| .last() | |||
| .and_then(|captures| choice_from_capture(&captures, 1)) | |||
| } | |||
| fn choice_from_capture(captures: ®ex::Captures<'_>, index: usize) -> Option<char> { | |||
| @@ -70,42 +63,26 @@ mod tests { | |||
| } | |||
| #[test] | |||
| fn extracts_choice_from_parenthesized_lowercase() { | |||
| assert_eq!(extract_choice("I choose (b)."), Some('B')); | |||
| fn extracts_choice_from_dollar_wrapped_answer() { | |||
| assert_eq!(extract_choice("Answer: $B$"), Some('B')); | |||
| } | |||
| #[test] | |||
| fn extracts_choice_from_final_answer_instead_of_earlier_prose_option() { | |||
| fn extracts_last_answer_prefix() { | |||
| assert_eq!( | |||
| extract_choice( | |||
| "Option A is tempting, but after checking the calculation the answer is D" | |||
| ), | |||
| extract_choice("Answer: A was my first thought.\nAnswer: D"), | |||
| Some('D') | |||
| ); | |||
| } | |||
| #[test] | |||
| fn extracts_choice_from_final_answer_after_considered_options() { | |||
| assert_eq!( | |||
| extract_choice("I considered A and B before deciding. Final answer: C"), | |||
| Some('C') | |||
| ); | |||
| fn ignores_choice_without_answer_prefix() { | |||
| assert_eq!(extract_choice("I choose (b)."), None); | |||
| } | |||
| #[test] | |||
| fn extracts_last_explicit_choice() { | |||
| assert_eq!( | |||
| extract_choice("Answer: A was my first thought. Final answer: D"), | |||
| Some('D') | |||
| ); | |||
| } | |||
| #[test] | |||
| fn ignores_prose_without_final_choice() { | |||
| assert_eq!( | |||
| extract_choice("This explanation has no final option."), | |||
| None | |||
| ); | |||
| fn ignores_bare_final_line_choice() { | |||
| assert_eq!(extract_choice("<think>...</think>\n\nC"), None); | |||
| } | |||
| #[test] | |||
| @@ -116,7 +93,7 @@ mod tests { | |||
| #[test] | |||
| fn judges_choice_by_extracted_value() { | |||
| assert!(judge_choice("I choose (b).", 'B')); | |||
| assert!(judge_choice("Answer: B", 'B')); | |||
| assert!(!judge_choice("Answer: C", 'B')); | |||
| } | |||
| } | |||
| @@ -138,7 +138,7 @@ pub enum BenchCommand { | |||
| limit: Option<usize>, | |||
| #[arg(long, num_args = 0..=1, default_missing_value = "true")] | |||
| stream: Option<bool>, | |||
| #[arg(long, default_value_t = 32768)] | |||
| #[arg(long, default_value_t = 20_000)] | |||
| max_tokens: u32, | |||
| #[arg(long)] | |||
| debug_raw: bool, | |||
| @@ -170,7 +170,7 @@ pub enum BenchCommand { | |||
| limit: Option<usize>, | |||
| #[arg(long, num_args = 0..=1, default_missing_value = "true")] | |||
| stream: Option<bool>, | |||
| #[arg(long, default_value_t = 32768)] | |||
| #[arg(long, default_value_t = 20_000)] | |||
| max_tokens: u32, | |||
| #[arg(long)] | |||
| debug_raw: bool, | |||
| @@ -1575,6 +1575,21 @@ mod tests { | |||
| assert_eq!(stream, Some(false)); | |||
| } | |||
| #[test] | |||
| fn bench_command_defaults_max_tokens_to_20000() { | |||
| let cli = Cli::try_parse_from(["lq_token_test", "bench", "gpqa-diamond"]) | |||
| .expect("parse gpqa bench"); | |||
| let Command::Bench { | |||
| command: BenchCommand::GpqaDiamond { max_tokens, .. }, | |||
| } = cli.command | |||
| else { | |||
| panic!("expected gpqa-diamond bench command"); | |||
| }; | |||
| assert_eq!(max_tokens, 20_000); | |||
| } | |||
| #[test] | |||
| fn check_command_parses_debug_raw() { | |||
| let cli = Cli::try_parse_from([ | |||