|
- use crate::benchmarks;
- use crate::config::AppConfig;
- use anyhow::Result;
- use clap::{Parser, Subcommand};
- use std::path::{Path, PathBuf};
-
- #[derive(Debug, Parser)]
- #[command(
- name = "lq_token_test",
- version,
- about = "Test LLM relay protocols, RPM, and benchmark accuracy"
- )]
- pub struct Cli {
- #[command(subcommand)]
- pub command: Command,
- }
-
- #[derive(Debug, Subcommand)]
- pub enum Command {
- Check {
- #[arg(long, default_value = "config.yaml")]
- config: PathBuf,
- #[arg(long)]
- provider: Option<String>,
- #[arg(long)]
- model: Option<String>,
- #[arg(long)]
- prompt: String,
- },
- Dataset {
- #[command(subcommand)]
- command: DatasetCommand,
- },
- Bench {
- #[command(subcommand)]
- command: BenchCommand,
- },
- Rpm {
- #[arg(long, default_value = "config.yaml")]
- config: PathBuf,
- #[arg(long)]
- provider: Option<String>,
- #[arg(long)]
- model: Option<String>,
- #[arg(long)]
- rpm: u32,
- #[arg(long)]
- duration: String,
- #[arg(long)]
- prompt: String,
- },
- }
-
- #[derive(Debug, Subcommand)]
- pub enum DatasetCommand {
- Fetch { dataset: String },
- }
-
- #[derive(Debug, Subcommand)]
- pub enum BenchCommand {
- Aime2026 {
- #[arg(long, default_value = "config.yaml")]
- config: PathBuf,
- #[arg(long)]
- provider: Option<String>,
- #[arg(long)]
- model: Option<String>,
- #[arg(long, default_value_t = 4)]
- concurrency: usize,
- #[arg(long)]
- limit: Option<usize>,
- },
- GpqaDiamond {
- #[arg(long, default_value = "config.yaml")]
- config: PathBuf,
- #[arg(long)]
- provider: Option<String>,
- #[arg(long)]
- model: Option<String>,
- #[arg(long, default_value_t = 4)]
- concurrency: usize,
- #[arg(long)]
- limit: Option<usize>,
- },
- }
-
- pub async fn dispatch(cli: Cli) -> Result<()> {
- match cli.command {
- Command::Check { .. } => anyhow::bail!("check is not implemented yet"),
- Command::Dataset {
- command: DatasetCommand::Fetch { dataset },
- } => {
- let data_dir = dataset_data_dir(Path::new("config.yaml"));
- let path = benchmarks::fetch_dataset(&dataset, &data_dir).await?;
- println!("{}", path.display());
- Ok(())
- }
- Command::Bench { .. } => anyhow::bail!("bench is not implemented yet"),
- Command::Rpm { .. } => anyhow::bail!("rpm is not implemented yet"),
- }
- }
-
- fn dataset_data_dir(config_path: &Path) -> PathBuf {
- if config_path.exists() {
- if let Ok(config) = AppConfig::load(config_path) {
- return PathBuf::from(config.benchmarks.data_dir);
- }
- }
-
- PathBuf::from("data/benchmarks")
- }
|