feat: prepare agen crates for publication
This commit is contained in:
@@ -0,0 +1,231 @@
|
||||
//! Test fixture recording tool
|
||||
//!
|
||||
//! Records API responses for defined scenarios.
|
||||
//!
|
||||
//! ## Usage
|
||||
//!
|
||||
//! ```bash
|
||||
//! # Show available scenarios
|
||||
//! cargo run --example record_test_fixtures
|
||||
//!
|
||||
//! # Record specific scenario
|
||||
//! ANTHROPIC_API_KEY=your-key cargo run --example record_test_fixtures -- simple_text
|
||||
//! ANTHROPIC_API_KEY=your-key cargo run --example record_test_fixtures -- tool_call
|
||||
//!
|
||||
//! # Record all scenarios
|
||||
//! ANTHROPIC_API_KEY=your-key cargo run --example record_test_fixtures -- --all
|
||||
//! ```
|
||||
|
||||
mod recorder;
|
||||
mod scenarios;
|
||||
|
||||
use agen::llm_client::scheme::{
|
||||
Scheme, anthropic::AnthropicScheme, gemini::GeminiScheme, openai_chat::OpenAIScheme,
|
||||
};
|
||||
use agen::llm_client::transport::{HttpTransport, ResolvedAuth};
|
||||
use clap::{Parser, ValueEnum};
|
||||
|
||||
fn make_transport<S: Scheme>(scheme: S, model: &str, auth: ResolvedAuth) -> HttpTransport<S> {
|
||||
let cap = scheme.default_capability();
|
||||
let base_url = scheme.default_base_url().to_string();
|
||||
HttpTransport::new(scheme, model.to_string(), base_url, auth, cap)
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(author, version, about, long_about = None)]
|
||||
struct Args {
|
||||
/// Scenario name
|
||||
#[arg(short, long)]
|
||||
scenario: Option<String>,
|
||||
|
||||
/// Run all scenarios
|
||||
#[arg(long, default_value_t = false)]
|
||||
all: bool,
|
||||
|
||||
/// Client to use
|
||||
#[arg(short, long, value_enum, default_value_t = ClientType::Anthropic)]
|
||||
client: ClientType,
|
||||
|
||||
/// Model to use (optional, defaults per client)
|
||||
#[arg(short, long)]
|
||||
model: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Debug)]
|
||||
enum ClientType {
|
||||
Anthropic,
|
||||
Gemini,
|
||||
Openai,
|
||||
Ollama,
|
||||
}
|
||||
|
||||
async fn run_scenario_with_anthropic(
|
||||
scenario: &scenarios::TestScenario,
|
||||
subdir: &str,
|
||||
model: Option<String>,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let api_key = std::env::var("ANTHROPIC_API_KEY")
|
||||
.expect("ANTHROPIC_API_KEY environment variable must be set");
|
||||
let model = model.as_deref().unwrap_or("claude-sonnet-4-20250514");
|
||||
let client = make_transport(AnthropicScheme::new(), model, ResolvedAuth::ApiKey(api_key));
|
||||
|
||||
recorder::record_request(
|
||||
&client,
|
||||
scenario.request.clone(),
|
||||
scenario.name,
|
||||
scenario.output_name,
|
||||
subdir,
|
||||
model,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_scenario_with_openai(
|
||||
scenario: &scenarios::TestScenario,
|
||||
subdir: &str,
|
||||
model: Option<String>,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let api_key =
|
||||
std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY environment variable must be set");
|
||||
let model = model.as_deref().unwrap_or("gpt-4o");
|
||||
let client = make_transport(OpenAIScheme::new(), model, ResolvedAuth::ApiKey(api_key));
|
||||
|
||||
recorder::record_request(
|
||||
&client,
|
||||
scenario.request.clone(),
|
||||
scenario.name,
|
||||
scenario.output_name,
|
||||
subdir,
|
||||
model,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_scenario_with_ollama(
|
||||
scenario: &scenarios::TestScenario,
|
||||
subdir: &str,
|
||||
model: Option<String>,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Ollama = Anthropic scheme + base_url 差し替え + 認証なし
|
||||
let model = model.as_deref().unwrap_or("llama3");
|
||||
let client = HttpTransport::new(
|
||||
AnthropicScheme::new(),
|
||||
model.to_string(),
|
||||
"http://localhost:11434".to_string(),
|
||||
ResolvedAuth::None,
|
||||
AnthropicScheme::new().default_capability(),
|
||||
);
|
||||
|
||||
recorder::record_request(
|
||||
&client,
|
||||
scenario.request.clone(),
|
||||
scenario.name,
|
||||
scenario.output_name,
|
||||
subdir,
|
||||
model,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_scenario_with_gemini(
|
||||
scenario: &scenarios::TestScenario,
|
||||
subdir: &str,
|
||||
model: Option<String>,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let api_key =
|
||||
std::env::var("GEMINI_API_KEY").expect("GEMINI_API_KEY environment variable must be set");
|
||||
let model = model.as_deref().unwrap_or("gemini-2.0-flash");
|
||||
let client = make_transport(GeminiScheme::new(), model, ResolvedAuth::ApiKey(api_key));
|
||||
|
||||
recorder::record_request(
|
||||
&client,
|
||||
scenario.request.clone(),
|
||||
scenario.name,
|
||||
scenario.output_name,
|
||||
subdir,
|
||||
model,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
dotenv::dotenv().ok();
|
||||
let args = Args::parse();
|
||||
|
||||
if !args.all && args.scenario.is_none() {
|
||||
use clap::CommandFactory;
|
||||
let mut cmd = Args::command();
|
||||
cmd.error(
|
||||
clap::error::ErrorKind::MissingRequiredArgument,
|
||||
"Either --all or --scenario <SCENARIO> must be provided",
|
||||
)
|
||||
.exit();
|
||||
}
|
||||
|
||||
let all_scenarios = scenarios::scenarios();
|
||||
|
||||
// Determine scenarios to run
|
||||
let scenarios_to_run: Vec<_> = if args.all {
|
||||
all_scenarios
|
||||
} else {
|
||||
let scenario_name = args.scenario.as_ref().unwrap();
|
||||
let found: Vec<_> = all_scenarios
|
||||
.into_iter()
|
||||
.filter(|s| s.output_name == scenario_name)
|
||||
.collect();
|
||||
|
||||
if found.is_empty() {
|
||||
eprintln!("Error: Unknown scenario '{}'", scenario_name);
|
||||
// Verify correct name by listing
|
||||
println!("Available scenarios:");
|
||||
for s in scenarios::scenarios() {
|
||||
println!(" {}", s.output_name);
|
||||
}
|
||||
std::process::exit(1);
|
||||
}
|
||||
found
|
||||
};
|
||||
|
||||
println!("=== Test Fixture Generator ===");
|
||||
println!("Client: {:?}", args.client);
|
||||
if let Some(ref m) = args.model {
|
||||
println!("Model: {}", m);
|
||||
}
|
||||
println!("Scenarios: {}\n", scenarios_to_run.len());
|
||||
|
||||
let subdir = match args.client {
|
||||
ClientType::Anthropic => "anthropic",
|
||||
ClientType::Gemini => "gemini",
|
||||
ClientType::Openai => "openai",
|
||||
ClientType::Ollama => "ollama",
|
||||
};
|
||||
|
||||
// Scenario filtering is already done in main.rs logic
|
||||
// Here we just execute in a simple loop
|
||||
for scenario in scenarios_to_run {
|
||||
match args.client {
|
||||
ClientType::Anthropic => {
|
||||
run_scenario_with_anthropic(&scenario, subdir, args.model.clone()).await?
|
||||
}
|
||||
ClientType::Gemini => {
|
||||
run_scenario_with_gemini(&scenario, subdir, args.model.clone()).await?
|
||||
}
|
||||
ClientType::Openai => {
|
||||
run_scenario_with_openai(&scenario, subdir, args.model.clone()).await?
|
||||
}
|
||||
ClientType::Ollama => {
|
||||
run_scenario_with_ollama(&scenario, subdir, args.model.clone()).await?
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("\n✅ Done!");
|
||||
println!("Run tests with: cargo test -p engine");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
//! Test fixture recording mechanism
|
||||
//!
|
||||
//! Saves events to files in JSONL format
|
||||
|
||||
use std::fs::{self, File};
|
||||
use std::io::{BufWriter, Write};
|
||||
use std::path::Path;
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use agen::llm_client::{LlmClient, Request};
|
||||
use futures::StreamExt;
|
||||
|
||||
/// Recorded event
|
||||
#[derive(Debug, serde::Serialize, serde::Deserialize)]
|
||||
pub struct RecordedEvent {
|
||||
pub elapsed_ms: u64,
|
||||
pub event_type: String,
|
||||
pub data: String,
|
||||
}
|
||||
|
||||
/// Session metadata
|
||||
#[derive(Debug, serde::Serialize, serde::Deserialize)]
|
||||
pub struct SessionMetadata {
|
||||
pub timestamp: u64,
|
||||
pub model: String,
|
||||
pub description: String,
|
||||
}
|
||||
|
||||
/// Save event sequence to file
|
||||
pub fn save_fixture(
|
||||
path: impl AsRef<Path>,
|
||||
metadata: &SessionMetadata,
|
||||
events: &[RecordedEvent],
|
||||
) -> std::io::Result<()> {
|
||||
let file = File::create(path)?;
|
||||
let mut writer = BufWriter::new(file);
|
||||
|
||||
writeln!(writer, "{}", serde_json::to_string(metadata)?)?;
|
||||
for event in events {
|
||||
writeln!(writer, "{}", serde_json::to_string(event)?)?;
|
||||
}
|
||||
writer.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Send request and record events
|
||||
pub async fn record_request<C: LlmClient>(
|
||||
client: &C,
|
||||
request: Request,
|
||||
description: &str,
|
||||
output_name: &str,
|
||||
subdir: &str, // e.g. "anthropic", "openai"
|
||||
model: &str,
|
||||
) -> Result<usize, Box<dyn std::error::Error>> {
|
||||
println!("\n📝 Recording: {}", description);
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut events: Vec<RecordedEvent> = Vec::new();
|
||||
|
||||
let mut stream = client.stream(request).await?;
|
||||
|
||||
while let Some(result) = stream.next().await {
|
||||
let elapsed = start_time.elapsed().as_millis() as u64;
|
||||
match result {
|
||||
Ok(event) => {
|
||||
let event_json = serde_json::to_string(&event)?;
|
||||
println!(" [{:>6}ms] {:?}", elapsed, event);
|
||||
events.push(RecordedEvent {
|
||||
elapsed_ms: elapsed,
|
||||
event_type: format!("{:?}", std::mem::discriminant(&event)),
|
||||
data: event_json,
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(" Error: {}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save
|
||||
let fixtures_dir = Path::new("engine/tests/fixtures").join(subdir);
|
||||
fs::create_dir_all(&fixtures_dir)?;
|
||||
|
||||
let filepath = fixtures_dir.join(format!("{}.jsonl", output_name));
|
||||
|
||||
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
|
||||
let metadata = SessionMetadata {
|
||||
timestamp,
|
||||
model: model.to_string(),
|
||||
description: description.to_string(),
|
||||
};
|
||||
|
||||
save_fixture(&filepath, &metadata, &events)?;
|
||||
|
||||
let event_count = events.len();
|
||||
println!(" 💾 Saved: {}", filepath.display());
|
||||
println!(" 📊 {} events recorded", event_count);
|
||||
|
||||
Ok(event_count)
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
//! Test fixture request definitions
|
||||
//!
|
||||
//! Defines requests and output file names for each scenario
|
||||
|
||||
use agen::llm_client::{Request, ToolDefinition};
|
||||
|
||||
/// Test scenario
|
||||
pub struct TestScenario {
|
||||
/// Scenario name (description)
|
||||
pub name: &'static str,
|
||||
/// Output file name (without extension)
|
||||
pub output_name: &'static str,
|
||||
/// Request
|
||||
pub request: Request,
|
||||
}
|
||||
|
||||
/// Get all test scenarios
|
||||
pub fn scenarios() -> Vec<TestScenario> {
|
||||
vec![
|
||||
simple_text_scenario(),
|
||||
tool_call_scenario(),
|
||||
long_text_scenario(),
|
||||
]
|
||||
}
|
||||
|
||||
/// Simple text response
|
||||
fn simple_text_scenario() -> TestScenario {
|
||||
TestScenario {
|
||||
name: "Simple text response",
|
||||
output_name: "simple_text",
|
||||
request: Request::new()
|
||||
.system("You are a helpful assistant. Be very concise.")
|
||||
.user("Say hello in one word.")
|
||||
.max_tokens(50),
|
||||
}
|
||||
}
|
||||
|
||||
/// Response with tool call
|
||||
fn tool_call_scenario() -> TestScenario {
|
||||
let get_weather_tool = ToolDefinition::new("get_weather")
|
||||
.description("Get the current weather for a city")
|
||||
.input_schema(serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string",
|
||||
"description": "The city name"
|
||||
}
|
||||
},
|
||||
"required": ["city"]
|
||||
}));
|
||||
|
||||
TestScenario {
|
||||
name: "Tool call response",
|
||||
output_name: "tool_call",
|
||||
request: Request::new()
|
||||
.system("You are a helpful assistant. Use tools when appropriate.")
|
||||
.user("What's the weather in Tokyo? Use the get_weather tool.")
|
||||
.tool(get_weather_tool)
|
||||
.max_tokens(200),
|
||||
}
|
||||
}
|
||||
|
||||
/// Long text generation scenario
|
||||
fn long_text_scenario() -> TestScenario {
|
||||
TestScenario {
|
||||
name: "Long text response",
|
||||
output_name: "long_text",
|
||||
request: Request::new()
|
||||
.system("You are a creative writer.")
|
||||
.user("Write a short story about a robot discovering a garden. It should be at least 300 words.")
|
||||
.max_tokens(1000),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user