api模型检测
選択できるのは25トピックまでです。 トピックは、先頭が英数字で、英数字とダッシュ('-')を使用した35文字以内のものにしてください。

235 行
7.8 KiB

  1. use anyhow::{Context, Result, bail};
  2. use std::collections::HashMap;
  3. use std::path::{Path, PathBuf};
  4. #[derive(Debug, Clone, PartialEq, Eq)]
  5. pub struct GpqaCase {
  6. pub id: String,
  7. pub question: String,
  8. pub choices: [(String, String); 4],
  9. pub answer: char,
  10. }
  11. #[derive(Debug, Clone, PartialEq, Eq)]
  12. pub struct LoadedGpqaCases {
  13. pub cases: Vec<GpqaCase>,
  14. pub local_path: PathBuf,
  15. }
  16. pub fn load_cases(data_dir: &Path) -> Result<LoadedGpqaCases> {
  17. let dataset_dir = data_dir.join("gpqa_diamond");
  18. let csv_path = dataset_dir.join("gpqa_diamond.csv");
  19. if !csv_path.exists() {
  20. bail!("missing local dataset; run lq_token_test dataset fetch gpqa-diamond");
  21. }
  22. let contents = std::fs::read_to_string(&csv_path)
  23. .with_context(|| format!("failed to read {}", csv_path.display()))?;
  24. let rows = parse_csv(&contents).context("failed to parse GPQA CSV")?;
  25. let (headers, records) = rows.split_first().context("GPQA CSV is empty")?;
  26. let header_index = headers
  27. .iter()
  28. .enumerate()
  29. .map(|(index, header)| (normalize_header(header), index))
  30. .collect::<HashMap<_, _>>();
  31. let cases = records
  32. .iter()
  33. .enumerate()
  34. .map(|(index, record)| {
  35. let question = csv_value(record, &header_index, &["question"])
  36. .with_context(|| format!("missing Question column in row {}", index + 2))?;
  37. let correct = csv_value(record, &header_index, &["correctanswer", "correct"])
  38. .with_context(|| format!("missing Correct Answer column in row {}", index + 2))?;
  39. let incorrect_1 = csv_value(record, &header_index, &["incorrectanswer1", "incorrect1"])
  40. .with_context(|| {
  41. format!("missing Incorrect Answer 1 column in row {}", index + 2)
  42. })?;
  43. let incorrect_2 = csv_value(record, &header_index, &["incorrectanswer2", "incorrect2"])
  44. .with_context(|| {
  45. format!("missing Incorrect Answer 2 column in row {}", index + 2)
  46. })?;
  47. let incorrect_3 = csv_value(record, &header_index, &["incorrectanswer3", "incorrect3"])
  48. .with_context(|| {
  49. format!("missing Incorrect Answer 3 column in row {}", index + 2)
  50. })?;
  51. let id = format!("gpqa-diamond-{}", index + 1);
  52. let (choices, answer) = rotated_choices(
  53. [
  54. correct.to_string(),
  55. incorrect_1.to_string(),
  56. incorrect_2.to_string(),
  57. incorrect_3.to_string(),
  58. ],
  59. index % 4,
  60. );
  61. Ok(GpqaCase {
  62. id,
  63. question: question.to_string(),
  64. choices,
  65. answer,
  66. })
  67. })
  68. .collect::<Result<Vec<_>>>()?;
  69. Ok(LoadedGpqaCases {
  70. cases,
  71. local_path: csv_path,
  72. })
  73. }
  74. fn csv_value<'a>(
  75. record: &'a [String],
  76. header_index: &HashMap<String, usize>,
  77. names: &[&str],
  78. ) -> Option<&'a str> {
  79. names
  80. .iter()
  81. .find_map(|name| header_index.get(*name))
  82. .and_then(|index| record.get(*index))
  83. .map(String::as_str)
  84. .filter(|value| !value.trim().is_empty())
  85. }
  86. fn normalize_header(header: &str) -> String {
  87. header
  88. .chars()
  89. .filter(|ch| ch.is_ascii_alphanumeric())
  90. .collect::<String>()
  91. .to_ascii_lowercase()
  92. }
  93. fn rotated_choices(mut values: [String; 4], rotation: usize) -> ([(String, String); 4], char) {
  94. let rotation = rotation % values.len();
  95. values.rotate_left(rotation);
  96. let answer_index = (4 - rotation) % 4;
  97. let labels = ["A", "B", "C", "D"];
  98. let answer = labels[answer_index]
  99. .chars()
  100. .next()
  101. .expect("labels are non-empty");
  102. let choices = std::array::from_fn(|index| (labels[index].to_string(), values[index].clone()));
  103. (choices, answer)
  104. }
  105. fn parse_csv(input: &str) -> Result<Vec<Vec<String>>> {
  106. let mut rows = Vec::new();
  107. let mut row = Vec::new();
  108. let mut field = String::new();
  109. let mut chars = input.chars().peekable();
  110. let mut in_quotes = false;
  111. while let Some(ch) = chars.next() {
  112. match ch {
  113. '"' if in_quotes && chars.peek() == Some(&'"') => {
  114. field.push('"');
  115. chars.next();
  116. }
  117. '"' => in_quotes = !in_quotes,
  118. ',' if !in_quotes => {
  119. row.push(std::mem::take(&mut field));
  120. }
  121. '\n' if !in_quotes => {
  122. row.push(std::mem::take(&mut field));
  123. if !row.iter().all(|value| value.is_empty()) {
  124. rows.push(std::mem::take(&mut row));
  125. } else {
  126. row.clear();
  127. }
  128. }
  129. '\r' if !in_quotes => {}
  130. _ => field.push(ch),
  131. }
  132. }
  133. if in_quotes {
  134. bail!("unterminated quoted field");
  135. }
  136. if !field.is_empty() || !row.is_empty() {
  137. row.push(field);
  138. rows.push(row);
  139. }
  140. Ok(rows)
  141. }
  142. impl GpqaCase {
  143. pub fn prompt(&self) -> String {
  144. let choices = self
  145. .choices
  146. .iter()
  147. .map(|(label, text)| format!("{label}. {text}"))
  148. .collect::<Vec<_>>()
  149. .join("\n");
  150. format!(
  151. "Answer the following multiple-choice question.\n\n{}\n\n{}\n\nPlease answer with exactly one letter: A, B, C, or D.",
  152. self.question, choices
  153. )
  154. }
  155. }
  156. #[cfg(test)]
  157. mod tests {
  158. use super::*;
  159. #[test]
  160. fn prompt_contains_choices_and_single_letter_instruction() {
  161. let case = GpqaCase {
  162. id: "gpqa-1".to_string(),
  163. question: "Which option is correct?".to_string(),
  164. choices: [
  165. ("A".to_string(), "Alpha".to_string()),
  166. ("B".to_string(), "Beta".to_string()),
  167. ("C".to_string(), "Gamma".to_string()),
  168. ("D".to_string(), "Delta".to_string()),
  169. ],
  170. answer: 'C',
  171. };
  172. let prompt = case.prompt();
  173. assert!(prompt.contains("Which option is correct?"));
  174. assert!(prompt.contains("A. Alpha"));
  175. assert!(prompt.contains("B. Beta"));
  176. assert!(prompt.contains("C. Gamma"));
  177. assert!(prompt.contains("D. Delta"));
  178. assert!(prompt.contains("answer with exactly one letter"));
  179. }
  180. #[test]
  181. fn loads_gpqa_csv_with_common_columns() {
  182. let temp_dir = tempfile::tempdir().expect("create temp dir");
  183. let dataset_dir = temp_dir.path().join("gpqa_diamond");
  184. std::fs::create_dir_all(&dataset_dir).expect("create dataset dir");
  185. std::fs::write(
  186. dataset_dir.join("gpqa_diamond.csv"),
  187. "Question,Correct Answer,Incorrect Answer 1,Incorrect Answer 2,Incorrect Answer 3\n\"Which is correct?\",\"Right\",\"Wrong, with comma\",\"Nope\",\"Never\"\n\"Which is second?\",\"Second right\",\"Second wrong 1\",\"Second wrong 2\",\"Second wrong 3\"\n",
  188. )
  189. .expect("write csv");
  190. let loaded = load_cases(temp_dir.path()).expect("load cases");
  191. assert_eq!(loaded.local_path, dataset_dir.join("gpqa_diamond.csv"));
  192. assert_eq!(loaded.cases.len(), 2);
  193. assert_eq!(loaded.cases[0].question, "Which is correct?");
  194. assert_eq!(
  195. loaded.cases[0].choices[0],
  196. ("A".to_string(), "Right".to_string())
  197. );
  198. assert_eq!(loaded.cases[0].answer, 'A');
  199. assert_eq!(loaded.cases[1].answer, 'D');
  200. assert_eq!(
  201. loaded.cases[1]
  202. .choices
  203. .iter()
  204. .find(|(label, _)| label == "D")
  205. .map(|(_, text)| text.as_str()),
  206. Some("Second right")
  207. );
  208. assert!(loaded.cases.iter().any(|case| case.answer != 'A'));
  209. }
  210. }