feat: error handling, retry with backoff, and circuit breaker
Add robust error handling to the evolution loop and LLM adapters: - Retry utility with exponential backoff for transient errors (429, 5xx, timeouts) - Per-call error isolation in evaluator and judge adapter - Circuit breaker in EvolutionLoop (trips after N consecutive failures) - CLI flags: --max-retries, --error-strategy (skip|retry|abort) - Config fields: max_retries, retry_delay_base, circuit_breaker_threshold, error_strategy - 16 new unit tests covering all error handling paths Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
@@ -56,6 +56,16 @@ def optimize(
|
||||
"--verbose",
|
||||
help="Print detailed progress.",
|
||||
),
|
||||
max_retries: int = typer.Option(
|
||||
3,
|
||||
"--max-retries",
|
||||
help="Max retry attempts for transient LLM errors (429, timeout, 5xx).",
|
||||
),
|
||||
error_strategy: str = typer.Option(
|
||||
"retry",
|
||||
"--error-strategy",
|
||||
help="How to handle errors: skip | retry | abort.",
|
||||
),
|
||||
) -> None:
|
||||
"""Optimize a prompt without any reference data.
|
||||
|
||||
@@ -115,6 +125,10 @@ def optimize(
|
||||
n_synthetic_inputs=raw_config.get("n_synthetic_inputs", 20),
|
||||
minibatch_size=raw_config.get("minibatch_size", 5),
|
||||
seed=raw_config.get("seed", 42),
|
||||
max_retries=raw_config.get("max_retries", max_retries),
|
||||
retry_delay_base=raw_config.get("retry_delay_base", 1.0),
|
||||
circuit_breaker_threshold=raw_config.get("circuit_breaker_threshold", 5),
|
||||
error_strategy=raw_config.get("error_strategy", error_strategy),
|
||||
output_path=output,
|
||||
verbose=verbose,
|
||||
)
|
||||
@@ -139,11 +153,23 @@ def optimize(
|
||||
**_model_lm_kwargs(config.synth_api_base, config.synth_api_key_env, global_api_base, global_api_key_env),
|
||||
)
|
||||
|
||||
# 3. Build adapters (Dependency Injection — each gets its own LM)
|
||||
# 3. Build adapters (Dependency Injection — each gets its own LM + retry config)
|
||||
synth_adapter = DSPySyntheticAdapter(lm=synth_lm)
|
||||
llm_adapter = DSPyLLMAdapter(lm=task_lm)
|
||||
judge_adapter = DSPyJudgeAdapter(lm=judge_lm)
|
||||
proposer_adapter = DSPyProposerAdapter(lm=proposer_lm)
|
||||
llm_adapter = DSPyLLMAdapter(
|
||||
lm=task_lm,
|
||||
max_retries=config.max_retries,
|
||||
retry_delay_base=config.retry_delay_base,
|
||||
)
|
||||
judge_adapter = DSPyJudgeAdapter(
|
||||
lm=judge_lm,
|
||||
max_retries=config.max_retries,
|
||||
retry_delay_base=config.retry_delay_base,
|
||||
)
|
||||
proposer_adapter = DSPyProposerAdapter(
|
||||
lm=proposer_lm,
|
||||
max_retries=config.max_retries,
|
||||
retry_delay_base=config.retry_delay_base,
|
||||
)
|
||||
bootstrap = SyntheticBootstrap(generator=synth_adapter, seed=config.seed)
|
||||
evaluator = PromptEvaluator(executor=llm_adapter, judge=judge_adapter)
|
||||
use_case = OptimizePromptUseCase(
|
||||
|
||||
Reference in New Issue
Block a user