64 lines
1.8 KiB
Python
64 lines
1.8 KiB
Python
"""Post-training evaluation over canonical scenarios."""
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from forja_core.api.deps import (
|
|
get_guardrail_engine,
|
|
get_llm_provider,
|
|
get_orchestrator,
|
|
get_policy_store,
|
|
get_registry,
|
|
get_settings,
|
|
)
|
|
from forja_core.evaluation.post_train import run_post_train_evaluation
|
|
from forja_core.evaluation.scenarios import load_agent_scenarios
|
|
|
|
FIXTURES = Path(__file__).parent.parent / "fixtures"
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def patch_settings(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
|
monkeypatch.setenv("AGENTS_DIR", str(FIXTURES / "agents"))
|
|
monkeypatch.setenv("POLICIES_DIR", str(FIXTURES / "policies"))
|
|
for fn in (
|
|
get_settings,
|
|
get_registry,
|
|
get_policy_store,
|
|
get_orchestrator,
|
|
get_llm_provider,
|
|
get_guardrail_engine,
|
|
):
|
|
fn.cache_clear()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_load_scenarios_from_examples() -> None:
|
|
scenarios = load_agent_scenarios(FIXTURES / "agents", "test_agent")
|
|
assert len(scenarios) >= 1
|
|
assert scenarios[0][0].endswith(".txt")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_train_eval_passes_for_test_agent() -> None:
|
|
registry = get_registry()
|
|
policies = get_policy_store()
|
|
orchestrator = get_orchestrator()
|
|
settings = get_settings()
|
|
agent = registry.get_agent("test_agent")
|
|
policy = policies.get_policy(agent.guardrails[0])
|
|
from uuid import uuid4
|
|
|
|
report = await run_post_train_evaluation(
|
|
training_run_id=uuid4(),
|
|
agent_def=agent,
|
|
policy=policy,
|
|
orchestrator=orchestrator,
|
|
agents_dir=settings.agents_dir,
|
|
)
|
|
assert report.scenario_count >= 1
|
|
assert report.passed is True
|
|
assert report.violation_count == 0
|