a5bf2ad59c
Aggregates all v0.2.0 sprint work (GARAA-30 through GARAA-40) and fixes 2 integration tests that broke when the codebase went async (DSPyLLMAdapter and full pipeline tests now properly await coroutines). 277 tests pass (260 unit + 17 integration). Co-Authored-By: Paperclip <noreply@paperclip.ing>
333 lines
12 KiB
Python
333 lines
12 KiB
Python
"""Unit tests for config and config loading via CLI.
|
|
|
|
Tests config validation scenarios: missing fields, wrong types, defaults.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
from pydantic import ValidationError
|
|
|
|
from prometheus.application.dto import OptimizationConfig
|
|
from prometheus.infrastructure.file_io import YamlPersistence
|
|
|
|
|
|
class TestOptimizationConfig:
|
|
"""Tests for OptimizationConfig Pydantic model defaults."""
|
|
|
|
def test_default_values(self) -> None:
|
|
config = OptimizationConfig(
|
|
seed_prompt="test prompt",
|
|
task_description="test task",
|
|
)
|
|
assert config.task_model == "openai/gpt-4o-mini"
|
|
assert config.judge_model == "openai/gpt-4o"
|
|
assert config.proposer_model == "openai/gpt-4o"
|
|
assert config.synth_model == "openai/gpt-4o"
|
|
assert config.max_iterations == 30
|
|
assert config.n_synthetic_inputs == 20
|
|
assert config.minibatch_size == 5
|
|
assert config.perfect_score == 1.0
|
|
assert config.seed == 42
|
|
assert config.output_path == "output.yaml"
|
|
assert config.verbose is False
|
|
assert config.error_strategy == "retry"
|
|
assert config.max_retries == 3
|
|
assert config.max_concurrency == 5
|
|
|
|
def test_custom_values(self) -> None:
|
|
config = OptimizationConfig(
|
|
seed_prompt="custom prompt",
|
|
task_description="custom task",
|
|
max_iterations=100,
|
|
minibatch_size=10,
|
|
seed=123,
|
|
verbose=True,
|
|
)
|
|
assert config.max_iterations == 100
|
|
assert config.minibatch_size == 10
|
|
assert config.seed == 123
|
|
assert config.verbose is True
|
|
|
|
def test_roundtrip_to_dict(self) -> None:
|
|
config = OptimizationConfig(
|
|
seed_prompt="test",
|
|
task_description="task",
|
|
)
|
|
d = config.model_dump()
|
|
assert d["seed_prompt"] == "test"
|
|
assert d["task_description"] == "task"
|
|
assert "history" not in d # OptimizationResult has history, not config
|
|
|
|
def test_config_version_defaults(self) -> None:
|
|
config = OptimizationConfig(seed_prompt="a", task_description="b")
|
|
assert config.config_version == 1
|
|
|
|
def test_config_version_stamps_current(self) -> None:
|
|
config = OptimizationConfig(
|
|
seed_prompt="a", task_description="b", config_version=0,
|
|
)
|
|
# Migration always stamps current version
|
|
assert config.config_version == 1
|
|
|
|
|
|
class TestConfigLoading:
|
|
"""Tests for loading OptimizationConfig from YAML via YamlPersistence."""
|
|
|
|
def test_minimal_config_loads(self, tmp_path: Path) -> None:
|
|
persistence = YamlPersistence()
|
|
data = {
|
|
"seed_prompt": "You are helpful.",
|
|
"task_description": "Answer questions.",
|
|
}
|
|
config_file = tmp_path / "config.yaml"
|
|
with open(config_file, "w") as f:
|
|
yaml.dump(data, f)
|
|
|
|
raw = persistence.read_config(str(config_file))
|
|
config = OptimizationConfig.model_validate(raw)
|
|
|
|
assert config.seed_prompt == "You are helpful."
|
|
assert config.task_description == "Answer questions."
|
|
assert config.max_iterations == 30 # default
|
|
|
|
def test_full_config_loads(self, tmp_path: Path) -> None:
|
|
persistence = YamlPersistence()
|
|
data = {
|
|
"seed_prompt": "You are helpful.",
|
|
"task_description": "Answer questions.",
|
|
"task_model": "openai/gpt-4o",
|
|
"judge_model": "openai/gpt-4o-mini",
|
|
"max_iterations": 50,
|
|
"n_synthetic_inputs": 30,
|
|
"minibatch_size": 8,
|
|
"seed": 99,
|
|
"verbose": True,
|
|
}
|
|
config_file = tmp_path / "config.yaml"
|
|
with open(config_file, "w") as f:
|
|
yaml.dump(data, f)
|
|
|
|
raw = persistence.read_config(str(config_file))
|
|
config = OptimizationConfig.model_validate(raw)
|
|
|
|
assert config.task_model == "openai/gpt-4o"
|
|
assert config.max_iterations == 50
|
|
assert config.verbose is True
|
|
|
|
def test_missing_seed_prompt_raises_validation_error(self, tmp_path: Path) -> None:
|
|
persistence = YamlPersistence()
|
|
data = {"task_description": "Answer questions."}
|
|
config_file = tmp_path / "config.yaml"
|
|
with open(config_file, "w") as f:
|
|
yaml.dump(data, f)
|
|
|
|
raw = persistence.read_config(str(config_file))
|
|
|
|
with pytest.raises(ValidationError, match="seed_prompt"):
|
|
OptimizationConfig.model_validate(raw)
|
|
|
|
def test_missing_task_description_raises_validation_error(self, tmp_path: Path) -> None:
|
|
persistence = YamlPersistence()
|
|
data = {"seed_prompt": "You are helpful."}
|
|
config_file = tmp_path / "config.yaml"
|
|
with open(config_file, "w") as f:
|
|
yaml.dump(data, f)
|
|
|
|
raw = persistence.read_config(str(config_file))
|
|
|
|
with pytest.raises(ValidationError, match="task_description"):
|
|
OptimizationConfig.model_validate(raw)
|
|
|
|
def test_empty_yaml_raises_on_required_fields(self, tmp_path: Path) -> None:
|
|
persistence = YamlPersistence()
|
|
config_file = tmp_path / "empty.yaml"
|
|
config_file.write_text("{}", encoding="utf-8")
|
|
|
|
raw = persistence.read_config(str(config_file))
|
|
|
|
with pytest.raises(ValidationError):
|
|
OptimizationConfig.model_validate(raw)
|
|
|
|
def test_partial_config_uses_defaults(self, tmp_path: Path) -> None:
|
|
persistence = YamlPersistence()
|
|
data = {
|
|
"seed_prompt": "test",
|
|
"task_description": "task",
|
|
"max_iterations": 10,
|
|
}
|
|
config_file = tmp_path / "partial.yaml"
|
|
with open(config_file, "w") as f:
|
|
yaml.dump(data, f)
|
|
|
|
raw = persistence.read_config(str(config_file))
|
|
config = OptimizationConfig.model_validate(raw)
|
|
|
|
assert config.max_iterations == 10
|
|
assert config.n_synthetic_inputs == 20 # default
|
|
assert config.minibatch_size == 5 # default
|
|
|
|
|
|
class TestConfigValidation:
|
|
"""Tests for Pydantic validation edge cases."""
|
|
|
|
def test_wrong_type_max_iterations(self) -> None:
|
|
with pytest.raises(ValidationError, match="max_iterations"):
|
|
OptimizationConfig(
|
|
seed_prompt="a",
|
|
task_description="b",
|
|
max_iterations="not_a_number", # type: ignore[arg-type]
|
|
)
|
|
|
|
def test_wrong_type_seed(self) -> None:
|
|
with pytest.raises(ValidationError, match="seed"):
|
|
OptimizationConfig(
|
|
seed_prompt="a",
|
|
task_description="b",
|
|
seed="not_a_number", # type: ignore[arg-type]
|
|
)
|
|
|
|
def test_negative_max_iterations(self) -> None:
|
|
with pytest.raises(ValidationError, match="greater than or equal to 1"):
|
|
OptimizationConfig(
|
|
seed_prompt="a", task_description="b", max_iterations=0,
|
|
)
|
|
|
|
def test_negative_seed(self) -> None:
|
|
with pytest.raises(ValidationError, match="greater than or equal to 0"):
|
|
OptimizationConfig(
|
|
seed_prompt="a", task_description="b", seed=-1,
|
|
)
|
|
|
|
def test_zero_minibatch_size(self) -> None:
|
|
with pytest.raises(ValidationError, match="greater than or equal to 1"):
|
|
OptimizationConfig(
|
|
seed_prompt="a", task_description="b", minibatch_size=0,
|
|
)
|
|
|
|
def test_zero_max_concurrency(self) -> None:
|
|
with pytest.raises(ValidationError, match="greater than or equal to 1"):
|
|
OptimizationConfig(
|
|
seed_prompt="a", task_description="b", max_concurrency=0,
|
|
)
|
|
|
|
def test_negative_max_retries(self) -> None:
|
|
with pytest.raises(ValidationError, match="greater than or equal to 0"):
|
|
OptimizationConfig(
|
|
seed_prompt="a", task_description="b", max_retries=-1,
|
|
)
|
|
|
|
def test_zero_retry_delay_base(self) -> None:
|
|
with pytest.raises(ValidationError, match="greater than 0"):
|
|
OptimizationConfig(
|
|
seed_prompt="a", task_description="b", retry_delay_base=0,
|
|
)
|
|
|
|
def test_negative_retry_delay_base(self) -> None:
|
|
with pytest.raises(ValidationError, match="greater than 0"):
|
|
OptimizationConfig(
|
|
seed_prompt="a", task_description="b", retry_delay_base=-0.5,
|
|
)
|
|
|
|
def test_zero_circuit_breaker_threshold(self) -> None:
|
|
with pytest.raises(ValidationError, match="greater than or equal to 1"):
|
|
OptimizationConfig(
|
|
seed_prompt="a", task_description="b", circuit_breaker_threshold=0,
|
|
)
|
|
|
|
def test_perfect_score_above_one(self) -> None:
|
|
with pytest.raises(ValidationError, match="less than or equal to 1"):
|
|
OptimizationConfig(
|
|
seed_prompt="a", task_description="b", perfect_score=1.5,
|
|
)
|
|
|
|
def test_perfect_score_negative(self) -> None:
|
|
with pytest.raises(ValidationError, match="greater than or equal to 0"):
|
|
OptimizationConfig(
|
|
seed_prompt="a", task_description="b", perfect_score=-0.1,
|
|
)
|
|
|
|
def test_invalid_error_strategy(self) -> None:
|
|
with pytest.raises(ValidationError, match="error_strategy must be one of"):
|
|
OptimizationConfig(
|
|
seed_prompt="a", task_description="b", error_strategy="invalid",
|
|
)
|
|
|
|
def test_valid_error_strategies(self) -> None:
|
|
for strategy in ("skip", "retry", "abort"):
|
|
config = OptimizationConfig(
|
|
seed_prompt="a", task_description="b", error_strategy=strategy,
|
|
)
|
|
assert config.error_strategy == strategy
|
|
|
|
def test_empty_seed_prompt(self) -> None:
|
|
with pytest.raises(ValidationError, match="seed_prompt"):
|
|
OptimizationConfig(seed_prompt="", task_description="b")
|
|
|
|
def test_empty_task_description(self) -> None:
|
|
with pytest.raises(ValidationError, match="task_description"):
|
|
OptimizationConfig(seed_prompt="a", task_description="")
|
|
|
|
def test_empty_model_string(self) -> None:
|
|
with pytest.raises(ValidationError, match="task_model"):
|
|
OptimizationConfig(
|
|
seed_prompt="a", task_description="b", task_model="",
|
|
)
|
|
|
|
def test_extra_fields_rejected(self) -> None:
|
|
with pytest.raises(ValidationError, match="extra"):
|
|
OptimizationConfig(
|
|
seed_prompt="a",
|
|
task_description="b",
|
|
nonexistent_field="value", # type: ignore[call-arg]
|
|
)
|
|
|
|
def test_boundary_values_accepted(self) -> None:
|
|
config = OptimizationConfig(
|
|
seed_prompt="a",
|
|
task_description="b",
|
|
max_iterations=1,
|
|
n_synthetic_inputs=1,
|
|
minibatch_size=1,
|
|
max_concurrency=1,
|
|
max_retries=0,
|
|
retry_delay_base=0.001,
|
|
circuit_breaker_threshold=1,
|
|
perfect_score=0.0,
|
|
seed=0,
|
|
)
|
|
assert config.max_iterations == 1
|
|
assert config.perfect_score == 0.0
|
|
|
|
|
|
class TestEvalConfigValidation:
|
|
"""Tests for ground-truth evaluation config fields."""
|
|
|
|
def test_eval_defaults(self) -> None:
|
|
config = OptimizationConfig(seed_prompt="a", task_description="b")
|
|
assert config.eval_dataset_path is None
|
|
assert config.eval_metric == "bleu"
|
|
|
|
def test_eval_dataset_path_set(self) -> None:
|
|
config = OptimizationConfig(
|
|
seed_prompt="a", task_description="b",
|
|
eval_dataset_path="data.csv",
|
|
)
|
|
assert config.eval_dataset_path == "data.csv"
|
|
|
|
def test_valid_eval_metrics(self) -> None:
|
|
for metric in ("exact", "bleu", "rouge_l", "cosine", "llm_judge"):
|
|
config = OptimizationConfig(
|
|
seed_prompt="a", task_description="b", eval_metric=metric,
|
|
)
|
|
assert config.eval_metric == metric
|
|
|
|
def test_invalid_eval_metric_raises(self) -> None:
|
|
with pytest.raises(ValidationError, match="eval_metric must be one of"):
|
|
OptimizationConfig(
|
|
seed_prompt="a", task_description="b",
|
|
eval_metric="invalid_metric",
|
|
)
|