c92ca4a2b8
Parallelize LLM calls across minibatches to reduce wall-clock time. All domain ports (LLMPort, JudgePort, ProposerPort) are now async. Adapter implementations wrap synchronous DSPy calls with asyncio.to_thread. Judge calls run in parallel within a batch using asyncio.gather + semaphore. Evaluator parallelizes minibatch execution with configurable concurrency. Evolution loop and use case are fully async. Proposer stays sequential. Added --max-concurrency CLI flag and max_concurrency YAML config field. Added async_retry_with_backoff for async error handling. All 139 unit tests pass. Co-Authored-By: Paperclip <noreply@paperclip.ing>
94 lines
2.3 KiB
Python
94 lines
2.3 KiB
Python
"""Shared test fixtures."""
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
from prometheus.domain.entities import (
|
|
EvalResult,
|
|
Prompt,
|
|
SyntheticExample,
|
|
Trajectory,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def seed_prompt() -> Prompt:
|
|
return Prompt(text="You are a helpful assistant. Answer the question.")
|
|
|
|
|
|
@pytest.fixture
|
|
def task_description() -> str:
|
|
return "Answer factual questions accurately and concisely."
|
|
|
|
|
|
@pytest.fixture
|
|
def synthetic_pool() -> list[SyntheticExample]:
|
|
return [
|
|
SyntheticExample(input_text=f"Test input {i}", id=i) for i in range(20)
|
|
]
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_eval_result() -> EvalResult:
|
|
return EvalResult(
|
|
scores=[0.3, 0.5, 0.4, 0.6, 0.2],
|
|
feedbacks=[
|
|
"Incomplete answer",
|
|
"Missing key detail",
|
|
"Wrong format",
|
|
"Partially correct",
|
|
"Completely off topic",
|
|
],
|
|
trajectories=[
|
|
Trajectory(
|
|
input_text=f"Input {i}",
|
|
output_text=f"Output {i}",
|
|
score=s,
|
|
feedback=f,
|
|
prompt_used="test prompt",
|
|
)
|
|
for i, (s, f) in enumerate(
|
|
zip(
|
|
[0.3, 0.5, 0.4, 0.6, 0.2],
|
|
[
|
|
"Incomplete answer",
|
|
"Missing key detail",
|
|
"Wrong format",
|
|
"Partially correct",
|
|
"Completely off topic",
|
|
],
|
|
)
|
|
)
|
|
],
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_llm_port() -> AsyncMock:
|
|
"""Mock LLMPort that returns canned responses."""
|
|
port = AsyncMock()
|
|
port.execute.return_value = "This is a mock response."
|
|
return port
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_judge_port() -> AsyncMock:
|
|
"""Mock JudgePort that returns moderate scores."""
|
|
port = AsyncMock()
|
|
port.judge_batch.return_value = [
|
|
(0.5, "Moderate quality, needs improvement."),
|
|
] * 5
|
|
return port
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_proposer_port() -> AsyncMock:
|
|
"""Mock ProposerPort that returns a slightly modified prompt."""
|
|
port = AsyncMock()
|
|
port.propose.return_value = Prompt(
|
|
text="You are a very helpful assistant. Answer the question precisely."
|
|
)
|
|
return port
|