Selaa lähdekoodia

chore: initialize cli project skeleton

main
orangels 1 kuukausi sitten
vanhempi
commit
c0ebdea077
16 muutettua tiedostoa jossa 2624 lisäystä ja 0 poistoa
  1. +3
    -0
      .gitignore
  2. +2446
    -0
      Cargo.lock
  3. +27
    -0
      Cargo.toml
  4. +23
    -0
      config.example.yaml
  5. +1
    -0
      src/benchmarks/aime.rs
  6. +1
    -0
      src/benchmarks/gpqa.rs
  7. +1
    -0
      src/benchmarks/judge.rs
  8. +3
    -0
      src/benchmarks/mod.rs
  9. +92
    -0
      src/cli.rs
  10. +1
    -0
      src/config.rs
  11. +20
    -0
      src/main.rs
  12. +1
    -0
      src/metrics.rs
  13. +1
    -0
      src/protocols/anthropic.rs
  14. +2
    -0
      src/protocols/mod.rs
  15. +1
    -0
      src/protocols/openai.rs
  16. +1
    -0
      src/runner.rs

+ 3
- 0
.gitignore Näytä tiedosto

@@ -0,0 +1,3 @@
/target
/data/benchmarks
/reports

+ 2446
- 0
Cargo.lock
File diff suppressed because it is too large
Näytä tiedosto


+ 27
- 0
Cargo.toml Näytä tiedosto

@@ -0,0 +1,27 @@
[package]
name = "lq_token_test"
version = "0.1.0"
edition = "2024"

[dependencies]
anyhow = "1"
chrono = { version = "0.4", features = ["serde"] }
clap = { version = "4", features = ["derive"] }
futures = "0.3"
hdrhistogram = "7"
indicatif = "0.17"
regex = "1"
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
serde_yaml = "0.9"
sha2 = "0.10"
thiserror = "2"
tokio = { version = "1", features = ["macros", "rt-multi-thread", "fs", "time"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
url = "2"

[dev-dependencies]
tempfile = "3"
wiremock = "0.6"

+ 23
- 0
config.example.yaml Näytä tiedosto

@@ -0,0 +1,23 @@
default_provider: openai

providers:
openai:
protocol: openai
base_url: "https://relay.example.com/v1"
api_token: "${OPENAI_RELAY_TOKEN}"
default_model: "gpt-4o-mini"

anthropic:
protocol: anthropic
base_url: "https://relay.example.com"
api_token: "${ANTHROPIC_RELAY_TOKEN}"
default_model: "claude-3-5-sonnet-latest"

benchmarks:
data_dir: "data/benchmarks"
aime2026:
source: "huggingface:MathArena/aime_2026"
split: "train"
gpqa_diamond:
source: "huggingface:Idavidrein/gpqa"
split: "gpqa_diamond"

+ 1
- 0
src/benchmarks/aime.rs Näytä tiedosto

@@ -0,0 +1 @@


+ 1
- 0
src/benchmarks/gpqa.rs Näytä tiedosto

@@ -0,0 +1 @@


+ 1
- 0
src/benchmarks/judge.rs Näytä tiedosto

@@ -0,0 +1 @@


+ 3
- 0
src/benchmarks/mod.rs Näytä tiedosto

@@ -0,0 +1,3 @@
pub mod aime;
pub mod gpqa;
pub mod judge;

+ 92
- 0
src/cli.rs Näytä tiedosto

@@ -0,0 +1,92 @@
use anyhow::Result;
use clap::{Parser, Subcommand};
use std::path::PathBuf;

#[derive(Debug, Parser)]
#[command(
name = "lq_token_test",
version,
about = "Test LLM relay protocols, RPM, and benchmark accuracy"
)]
pub struct Cli {
#[command(subcommand)]
pub command: Command,
}

#[derive(Debug, Subcommand)]
pub enum Command {
Check {
#[arg(long, default_value = "config.yaml")]
config: PathBuf,
#[arg(long)]
provider: Option<String>,
#[arg(long)]
model: Option<String>,
#[arg(long)]
prompt: String,
},
Dataset {
#[command(subcommand)]
command: DatasetCommand,
},
Bench {
#[command(subcommand)]
command: BenchCommand,
},
Rpm {
#[arg(long, default_value = "config.yaml")]
config: PathBuf,
#[arg(long)]
provider: Option<String>,
#[arg(long)]
model: Option<String>,
#[arg(long)]
rpm: u32,
#[arg(long)]
duration: String,
#[arg(long)]
prompt: String,
},
}

#[derive(Debug, Subcommand)]
pub enum DatasetCommand {
Fetch { dataset: String },
}

#[derive(Debug, Subcommand)]
pub enum BenchCommand {
Aime2026 {
#[arg(long, default_value = "config.yaml")]
config: PathBuf,
#[arg(long)]
provider: Option<String>,
#[arg(long)]
model: Option<String>,
#[arg(long, default_value_t = 4)]
concurrency: usize,
#[arg(long)]
limit: Option<usize>,
},
GpqaDiamond {
#[arg(long, default_value = "config.yaml")]
config: PathBuf,
#[arg(long)]
provider: Option<String>,
#[arg(long)]
model: Option<String>,
#[arg(long, default_value_t = 4)]
concurrency: usize,
#[arg(long)]
limit: Option<usize>,
},
}

pub async fn dispatch(cli: Cli) -> Result<()> {
match cli.command {
Command::Check { .. } => anyhow::bail!("check is not implemented yet"),
Command::Dataset { .. } => anyhow::bail!("dataset is not implemented yet"),
Command::Bench { .. } => anyhow::bail!("bench is not implemented yet"),
Command::Rpm { .. } => anyhow::bail!("rpm is not implemented yet"),
}
}

+ 1
- 0
src/config.rs Näytä tiedosto

@@ -0,0 +1 @@


+ 20
- 0
src/main.rs Näytä tiedosto

@@ -0,0 +1,20 @@
mod benchmarks;
mod cli;
mod config;
mod metrics;
mod protocols;
mod runner;

use anyhow::Result;
use clap::Parser;
use cli::Cli;

#[tokio::main]
async fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.init();

let cli = Cli::parse();
cli::dispatch(cli).await
}

+ 1
- 0
src/metrics.rs Näytä tiedosto

@@ -0,0 +1 @@


+ 1
- 0
src/protocols/anthropic.rs Näytä tiedosto

@@ -0,0 +1 @@


+ 2
- 0
src/protocols/mod.rs Näytä tiedosto

@@ -0,0 +1,2 @@
pub mod anthropic;
pub mod openai;

+ 1
- 0
src/protocols/openai.rs Näytä tiedosto

@@ -0,0 +1 @@


+ 1
- 0
src/runner.rs Näytä tiedosto

@@ -0,0 +1 @@


Ladataan…
Peruuta
Tallenna