feat: error handling, retry with backoff, and circuit breaker

Add robust error handling to the evolution loop and LLM adapters:
- Retry utility with exponential backoff for transient errors (429, 5xx, timeouts)
- Per-call error isolation in evaluator and judge adapter
- Circuit breaker in EvolutionLoop (trips after N consecutive failures)
- CLI flags: --max-retries, --error-strategy (skip|retry|abort)
- Config fields: max_retries, retry_delay_base, circuit_breaker_threshold, error_strategy
- 16 new unit tests covering all error handling paths

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
FullStackDev
2026-03-29 12:47:55 +00:00
parent f516ca4be6
commit e2d111ce5b
10 changed files with 646 additions and 103 deletions

View File

@@ -56,6 +56,16 @@ def optimize(
"--verbose",
help="Print detailed progress.",
),
max_retries: int = typer.Option(
3,
"--max-retries",
help="Max retry attempts for transient LLM errors (429, timeout, 5xx).",
),
error_strategy: str = typer.Option(
"retry",
"--error-strategy",
help="How to handle errors: skip | retry | abort.",
),
) -> None:
"""Optimize a prompt without any reference data.
@@ -115,6 +125,10 @@ def optimize(
n_synthetic_inputs=raw_config.get("n_synthetic_inputs", 20),
minibatch_size=raw_config.get("minibatch_size", 5),
seed=raw_config.get("seed", 42),
max_retries=raw_config.get("max_retries", max_retries),
retry_delay_base=raw_config.get("retry_delay_base", 1.0),
circuit_breaker_threshold=raw_config.get("circuit_breaker_threshold", 5),
error_strategy=raw_config.get("error_strategy", error_strategy),
output_path=output,
verbose=verbose,
)
@@ -139,11 +153,23 @@ def optimize(
**_model_lm_kwargs(config.synth_api_base, config.synth_api_key_env, global_api_base, global_api_key_env),
)
# 3. Build adapters (Dependency Injection — each gets its own LM)
# 3. Build adapters (Dependency Injection — each gets its own LM + retry config)
synth_adapter = DSPySyntheticAdapter(lm=synth_lm)
llm_adapter = DSPyLLMAdapter(lm=task_lm)
judge_adapter = DSPyJudgeAdapter(lm=judge_lm)
proposer_adapter = DSPyProposerAdapter(lm=proposer_lm)
llm_adapter = DSPyLLMAdapter(
lm=task_lm,
max_retries=config.max_retries,
retry_delay_base=config.retry_delay_base,
)
judge_adapter = DSPyJudgeAdapter(
lm=judge_lm,
max_retries=config.max_retries,
retry_delay_base=config.retry_delay_base,
)
proposer_adapter = DSPyProposerAdapter(
lm=proposer_lm,
max_retries=config.max_retries,
retry_delay_base=config.retry_delay_base,
)
bootstrap = SyntheticBootstrap(generator=synth_adapter, seed=config.seed)
evaluator = PromptEvaluator(executor=llm_adapter, judge=judge_adapter)
use_case = OptimizePromptUseCase(