e2d111ce5b
Add robust error handling to the evolution loop and LLM adapters: - Retry utility with exponential backoff for transient errors (429, 5xx, timeouts) - Per-call error isolation in evaluator and judge adapter - Circuit breaker in EvolutionLoop (trips after N consecutive failures) - CLI flags: --max-retries, --error-strategy (skip|retry|abort) - Config fields: max_retries, retry_delay_base, circuit_breaker_threshold, error_strategy - 16 new unit tests covering all error handling paths Co-Authored-By: Paperclip <noreply@paperclip.ing>
64 lines
1.7 KiB
Python
64 lines
1.7 KiB
Python
"""Data Transfer Objects — configuration and results."""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
|
|
@dataclass
|
|
class OptimizationConfig:
|
|
"""Complete configuration for a PROMETHEUS run."""
|
|
|
|
# --- Prompt ---
|
|
seed_prompt: str
|
|
task_description: str
|
|
|
|
# --- Models ---
|
|
task_model: str = "openai/gpt-4o-mini"
|
|
judge_model: str = "openai/gpt-4o"
|
|
proposer_model: str = "openai/gpt-4o"
|
|
synth_model: str = "openai/gpt-4o"
|
|
|
|
# --- Per-model API overrides (optional, fall back to global api_base/api_key_env) ---
|
|
task_api_base: str | None = None
|
|
task_api_key_env: str | None = None
|
|
judge_api_base: str | None = None
|
|
judge_api_key_env: str | None = None
|
|
proposer_api_base: str | None = None
|
|
proposer_api_key_env: str | None = None
|
|
synth_api_base: str | None = None
|
|
synth_api_key_env: str | None = None
|
|
|
|
# --- Evolution parameters ---
|
|
max_iterations: int = 30
|
|
n_synthetic_inputs: int = 20
|
|
minibatch_size: int = 5
|
|
perfect_score: float = 1.0
|
|
|
|
# --- Reproducibility ---
|
|
seed: int = 42
|
|
|
|
# --- Error handling ---
|
|
max_retries: int = 3
|
|
retry_delay_base: float = 1.0
|
|
circuit_breaker_threshold: int = 5
|
|
error_strategy: str = "retry" # skip | retry | abort
|
|
|
|
# --- Output ---
|
|
output_path: str = "output.yaml"
|
|
verbose: bool = False
|
|
|
|
|
|
@dataclass
|
|
class OptimizationResult:
|
|
"""Result of a complete optimization."""
|
|
|
|
optimized_prompt: str
|
|
initial_prompt: str
|
|
iterations_used: int
|
|
total_llm_calls: int
|
|
initial_score: float
|
|
final_score: float
|
|
improvement: float
|
|
history: list[dict[str, Any]] = field(default_factory=list)
|