pagina unica
This commit is contained in:
@@ -0,0 +1 @@
|
||||
Routine platform health check. No user impact reported.
|
||||
+7
-2
@@ -3,6 +3,11 @@ versions:
|
||||
- id: v1
|
||||
hash: feedface
|
||||
author: pytest
|
||||
message: fixture inicial
|
||||
message: fixture initial
|
||||
created_at: 2026-01-01T00:00:00Z
|
||||
active_version: v1
|
||||
- id: v2
|
||||
hash: draftv2
|
||||
author: pytest
|
||||
message: draft for promotion tests
|
||||
created_at: 2026-06-01T00:00:00Z
|
||||
active_version: v1
|
||||
@@ -0,0 +1,18 @@
|
||||
name: test_agent
|
||||
version: v2
|
||||
owner: pytest
|
||||
purpose: Draft version for governance promotion tests.
|
||||
state: draft
|
||||
guardrails: [test_policy]
|
||||
llm:
|
||||
provider: mock
|
||||
model: gpt-4o
|
||||
temperature: 0.2
|
||||
max_tokens: 2000
|
||||
system_prompt: |
|
||||
You are a test agent. Return JSON with severity and proposed_actions.
|
||||
output_schema:
|
||||
type: object
|
||||
required: [severity]
|
||||
risk_threshold_for_hitl: 4
|
||||
updated_at: 2026-06-01T00:00:00Z
|
||||
@@ -0,0 +1,8 @@
|
||||
name: incident_sft
|
||||
versions:
|
||||
- id: v1
|
||||
hash: pending
|
||||
author: test
|
||||
message: fixture
|
||||
created_at: 2026-01-01T00:00:00Z
|
||||
active_version: v1
|
||||
@@ -0,0 +1,8 @@
|
||||
name: incident_sft
|
||||
version: v1
|
||||
owner: test
|
||||
purpose: Test dataset
|
||||
state: active
|
||||
format: jsonl
|
||||
records_path: records/v1.jsonl
|
||||
updated_at: 2026-01-01T00:00:00Z
|
||||
@@ -0,0 +1,8 @@
|
||||
name: gpt4o_lora_base
|
||||
versions:
|
||||
- id: v1
|
||||
hash: pending
|
||||
author: test
|
||||
message: fixture
|
||||
created_at: 2026-01-01T00:00:00Z
|
||||
active_version: v1
|
||||
@@ -0,0 +1,8 @@
|
||||
name: gpt4o_lora_base
|
||||
version: v1
|
||||
owner: test
|
||||
purpose: Test model
|
||||
state: active
|
||||
base_model_id: gpt-4o
|
||||
adaptation: lora
|
||||
updated_at: 2026-01-01T00:00:00Z
|
||||
@@ -4,9 +4,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
EXAMPLES = (
|
||||
Path(__file__).parent.parent.parent / "agents" / "incident_analyzer" / "examples"
|
||||
)
|
||||
EXAMPLES = Path(__file__).parent.parent.parent / "agents" / "incident_analyzer" / "examples"
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
|
||||
@@ -4,9 +4,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
EXAMPLES = (
|
||||
Path(__file__).parent.parent.parent / "agents" / "incident_analyzer" / "examples"
|
||||
)
|
||||
EXAMPLES = Path(__file__).parent.parent.parent / "agents" / "incident_analyzer" / "examples"
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
|
||||
@@ -10,9 +10,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
EXAMPLES = (
|
||||
Path(__file__).parent.parent.parent / "agents" / "incident_analyzer" / "examples"
|
||||
)
|
||||
EXAMPLES = Path(__file__).parent.parent.parent / "agents" / "incident_analyzer" / "examples"
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
|
||||
@@ -34,7 +34,9 @@ def client() -> TestClient:
|
||||
|
||||
|
||||
def test_invoke_devuelve_execution_completada(client: TestClient) -> None:
|
||||
r = client.post("/api/agents/incident_analyzer/invoke", json={"input": "degradación MOS pool SBC"})
|
||||
r = client.post(
|
||||
"/api/agents/incident_analyzer/invoke", json={"input": "degradación MOS pool SBC"}
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["status"] == "completed"
|
||||
|
||||
@@ -59,7 +59,9 @@ def test_violations_vacio_si_no_hay(client: TestClient) -> None:
|
||||
assert r.json() == []
|
||||
|
||||
|
||||
def _violation(trace_id: object, validator: str, severity: str, blocked: bool) -> GuardrailViolation:
|
||||
def _violation(
|
||||
trace_id: object, validator: str, severity: str, blocked: bool
|
||||
) -> GuardrailViolation:
|
||||
return GuardrailViolation(
|
||||
trace_id=trace_id, # type: ignore[arg-type]
|
||||
timestamp=datetime.now(UTC),
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Governance promotion flow: train → evaluate → request → approve."""
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from forja_core.api import deps
|
||||
from forja_core.main import create_app
|
||||
|
||||
FIXTURES = Path(__file__).parent.parent / "fixtures"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def patch_settings(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
agents_dir = tmp_path / "agents"
|
||||
shutil.copytree(FIXTURES / "agents", agents_dir)
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
||||
monkeypatch.setenv("AGENTS_DIR", str(agents_dir))
|
||||
monkeypatch.setenv("POLICIES_DIR", str(FIXTURES / "policies"))
|
||||
monkeypatch.setenv("DATASETS_DIR", str(FIXTURES / "datasets"))
|
||||
monkeypatch.setenv("MODELS_DIR", str(FIXTURES / "models"))
|
||||
monkeypatch.setenv("TRAINING_BACKEND", "mock")
|
||||
for fn in (
|
||||
deps.get_settings,
|
||||
deps.get_registry,
|
||||
deps.get_policy_store,
|
||||
deps.get_dataset_store,
|
||||
deps.get_model_store,
|
||||
deps.get_training_backend,
|
||||
deps.get_orchestrator,
|
||||
deps.get_llm_provider,
|
||||
deps.get_guardrail_engine,
|
||||
):
|
||||
fn.cache_clear()
|
||||
|
||||
|
||||
def test_promotion_flow_approve_activates_draft_version() -> None:
|
||||
client = TestClient(create_app())
|
||||
|
||||
run = client.post(
|
||||
"/api/training/runs",
|
||||
json={
|
||||
"dataset_name": "incident_sft",
|
||||
"model_name": "gpt4o_lora_base",
|
||||
"agent_name": "test_agent",
|
||||
"agent_version": "v2",
|
||||
},
|
||||
).json()
|
||||
run_id = run["id"]
|
||||
|
||||
# Evaluation must cover the candidate version, not the active one.
|
||||
eval_report = client.post(f"/api/training/runs/{run_id}/evaluate").json()
|
||||
assert eval_report["passed"] is True
|
||||
assert eval_report["agent_version"] == "v2"
|
||||
|
||||
promo = client.post(
|
||||
"/api/promotions",
|
||||
json={
|
||||
"agent_name": "test_agent",
|
||||
"agent_version": "v2",
|
||||
"training_run_id": run_id,
|
||||
"evaluation_report_id": eval_report["id"],
|
||||
},
|
||||
)
|
||||
assert promo.status_code == 201
|
||||
request_id = promo.json()["id"]
|
||||
|
||||
approve = client.post(
|
||||
f"/api/promotions/{request_id}/approve",
|
||||
json={"approved_by": "ops-lead", "comment": "Eval gate passed"},
|
||||
)
|
||||
assert approve.status_code == 200
|
||||
|
||||
agent = client.get("/api/agents/test_agent").json()
|
||||
assert agent["version"] == "v2"
|
||||
assert agent["state"] == "active"
|
||||
|
||||
|
||||
def test_promotion_blocked_when_eval_failed(tmp_path: Path) -> None:
|
||||
"""Promotion returns 422 if linked evaluation did not pass."""
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from forja_core.api.evaluation_persistence import save_evaluation
|
||||
from forja_core.api.training_persistence import save_training_run
|
||||
from forja_core.domain.training import (
|
||||
EvaluationReport,
|
||||
TrainingHyperparameters,
|
||||
TrainingJobSpec,
|
||||
TrainingRun,
|
||||
)
|
||||
|
||||
client = TestClient(create_app())
|
||||
settings = deps.get_settings()
|
||||
run_id = uuid4()
|
||||
now = datetime.now(UTC)
|
||||
spec = TrainingJobSpec(
|
||||
dataset_name="incident_sft",
|
||||
dataset_version="v1",
|
||||
model_name="gpt4o_lora_base",
|
||||
model_version="v1",
|
||||
agent_name="test_agent",
|
||||
hyperparameters=TrainingHyperparameters(),
|
||||
)
|
||||
save_training_run(
|
||||
settings.data_dir,
|
||||
TrainingRun(
|
||||
id=run_id,
|
||||
backend="mock",
|
||||
status="succeeded",
|
||||
spec=spec,
|
||||
external_job_id="mock:x",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
),
|
||||
)
|
||||
report = EvaluationReport(
|
||||
id=uuid4(),
|
||||
training_run_id=run_id,
|
||||
agent_name="test_agent",
|
||||
agent_version="v1",
|
||||
passed=False,
|
||||
scenario_count=1,
|
||||
violation_count=1,
|
||||
created_at=now,
|
||||
notes="forced failure",
|
||||
)
|
||||
save_evaluation(settings.data_dir, report)
|
||||
|
||||
r = client.post(
|
||||
"/api/promotions",
|
||||
json={
|
||||
"agent_name": "test_agent",
|
||||
"agent_version": "v2",
|
||||
"training_run_id": str(run_id),
|
||||
"evaluation_report_id": str(report.id),
|
||||
},
|
||||
)
|
||||
assert r.status_code == 422
|
||||
|
||||
|
||||
def test_promotion_blocked_when_eval_covers_other_version() -> None:
|
||||
"""Evaluating the active version must not unlock promoting a different draft."""
|
||||
client = TestClient(create_app())
|
||||
|
||||
# No agent_version on the run → evaluation covers the active version (v1).
|
||||
run = client.post(
|
||||
"/api/training/runs",
|
||||
json={
|
||||
"dataset_name": "incident_sft",
|
||||
"model_name": "gpt4o_lora_base",
|
||||
"agent_name": "test_agent",
|
||||
},
|
||||
).json()
|
||||
run_id = run["id"]
|
||||
|
||||
eval_report = client.post(f"/api/training/runs/{run_id}/evaluate").json()
|
||||
assert eval_report["passed"] is True
|
||||
assert eval_report["agent_version"] == "v1"
|
||||
|
||||
r = client.post(
|
||||
"/api/promotions",
|
||||
json={
|
||||
"agent_name": "test_agent",
|
||||
"agent_version": "v2",
|
||||
"training_run_id": run_id,
|
||||
"evaluation_report_id": eval_report["id"],
|
||||
},
|
||||
)
|
||||
assert r.status_code == 422
|
||||
assert "evaluate the candidate version" in r.json()["detail"]
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Tests for /api/training/runs and registry endpoints."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from forja_core.api import deps
|
||||
from forja_core.main import create_app
|
||||
|
||||
FIXTURES = Path(__file__).parent.parent / "fixtures"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def patch_settings(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
||||
monkeypatch.setenv("AGENTS_DIR", str(FIXTURES / "agents"))
|
||||
monkeypatch.setenv("POLICIES_DIR", str(FIXTURES / "policies"))
|
||||
monkeypatch.setenv("DATASETS_DIR", str(FIXTURES / "datasets"))
|
||||
monkeypatch.setenv("MODELS_DIR", str(FIXTURES / "models"))
|
||||
monkeypatch.setenv("TRAINING_BACKEND", "mock")
|
||||
for fn in (
|
||||
deps.get_settings,
|
||||
deps.get_registry,
|
||||
deps.get_policy_store,
|
||||
deps.get_dataset_store,
|
||||
deps.get_model_store,
|
||||
deps.get_training_backend,
|
||||
deps.get_orchestrator,
|
||||
deps.get_llm_provider,
|
||||
deps.get_guardrail_engine,
|
||||
):
|
||||
fn.cache_clear()
|
||||
|
||||
|
||||
def test_list_datasets() -> None:
|
||||
r = TestClient(create_app()).get("/api/datasets")
|
||||
assert r.status_code == 200
|
||||
assert any(d["name"] == "incident_sft" for d in r.json())
|
||||
|
||||
|
||||
def test_create_training_run_mock_backend(tmp_path: Path) -> None:
|
||||
client = TestClient(create_app())
|
||||
r = client.post(
|
||||
"/api/training/runs",
|
||||
json={
|
||||
"dataset_name": "incident_sft",
|
||||
"model_name": "gpt4o_lora_base",
|
||||
},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
body = r.json()
|
||||
assert body["status"] == "succeeded"
|
||||
assert body["backend"] == "mock"
|
||||
assert body["external_job_id"].startswith("mock:")
|
||||
assert len(body["artifact_refs"]) >= 1
|
||||
run_id = body["id"]
|
||||
assert (tmp_path / "training_runs" / f"{run_id}.json").exists()
|
||||
|
||||
r2 = client.get(f"/api/training/runs/{run_id}")
|
||||
assert r2.status_code == 200
|
||||
assert r2.json()["id"] == run_id
|
||||
|
||||
|
||||
def test_create_training_run_unknown_dataset_404() -> None:
|
||||
r = TestClient(create_app()).post(
|
||||
"/api/training/runs",
|
||||
json={"dataset_name": "missing", "model_name": "gpt4o_lora_base"},
|
||||
)
|
||||
assert r.status_code == 404
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Azure ML SDK backend with mocked MLClient."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from forja_core.config import Settings
|
||||
from forja_core.domain.training import TrainingHyperparameters, TrainingJobSpec
|
||||
from forja_core.training.azure_ml import AzureMLTrainingBackend
|
||||
from forja_core.training.azure_ml_sdk import poll_job_status, submit_command_job
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def azure_settings() -> Settings:
|
||||
return Settings(
|
||||
training_backend="azure_ml",
|
||||
azure_ml_subscription_id="sub-123",
|
||||
azure_ml_resource_group="rg-forja",
|
||||
azure_ml_workspace_name="ws-forja",
|
||||
azure_ml_compute="cpu-cluster",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def spec() -> TrainingJobSpec:
|
||||
return TrainingJobSpec(
|
||||
dataset_name="incident_sft",
|
||||
dataset_version="v1",
|
||||
model_name="gpt4o_lora_base",
|
||||
model_version="v1",
|
||||
agent_name="test_agent",
|
||||
hyperparameters=TrainingHyperparameters(),
|
||||
)
|
||||
|
||||
|
||||
@patch("forja_core.training.azure_ml_sdk._ml_client")
|
||||
def test_submit_command_job_returns_azureml_prefix(
|
||||
mock_client_factory: MagicMock,
|
||||
azure_settings: Settings,
|
||||
spec: TrainingJobSpec,
|
||||
) -> None:
|
||||
client = MagicMock()
|
||||
mock_client_factory.return_value = client
|
||||
created = MagicMock()
|
||||
created.name = "forja-job-abc"
|
||||
client.jobs.create_or_update.return_value = created
|
||||
|
||||
run_id = uuid4()
|
||||
external_id = submit_command_job(azure_settings, spec, run_id)
|
||||
|
||||
assert external_id == "azureml:forja-job-abc"
|
||||
client.jobs.create_or_update.assert_called_once()
|
||||
|
||||
|
||||
@patch("forja_core.training.azure_ml_sdk._ml_client")
|
||||
def test_poll_job_status_maps_completed(
|
||||
mock_client_factory: MagicMock,
|
||||
azure_settings: Settings,
|
||||
) -> None:
|
||||
client = MagicMock()
|
||||
mock_client_factory.return_value = client
|
||||
job = MagicMock()
|
||||
job.status = "Completed"
|
||||
client.jobs.get.return_value = job
|
||||
|
||||
result = poll_job_status(azure_settings, "forja-job-abc")
|
||||
|
||||
assert result.status == "succeeded"
|
||||
assert len(result.artifacts) == 1
|
||||
assert "azureml://" in result.artifacts[0].uri
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backend_uses_sdk_when_configured(
|
||||
azure_settings: Settings,
|
||||
spec: TrainingJobSpec,
|
||||
) -> None:
|
||||
backend = AzureMLTrainingBackend(azure_settings)
|
||||
assert backend._use_sdk is True
|
||||
|
||||
with patch(
|
||||
"forja_core.training.azure_ml_sdk.submit_command_job",
|
||||
return_value="azureml:job-1",
|
||||
) as mock_submit:
|
||||
job_id = await backend.submit(spec, uuid4())
|
||||
assert job_id == "azureml:job-1"
|
||||
mock_submit.assert_called_once()
|
||||
|
||||
with patch(
|
||||
"forja_core.training.azure_ml_sdk.poll_job_status",
|
||||
return_value=MagicMock(status="running", message=None, artifacts=[]),
|
||||
) as mock_poll:
|
||||
status = await backend.status("azureml:job-1")
|
||||
assert status.status == "running"
|
||||
mock_poll.assert_called_once_with(azure_settings, "job-1")
|
||||
@@ -1,9 +1,13 @@
|
||||
"""Tests del factory LLM."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from forja_core.config import Settings
|
||||
from forja_core.llm.base import CompletionResult, Message
|
||||
from forja_core.llm.factory import build_llm_provider
|
||||
from forja_core.llm.fallback import FallbackLLMProvider
|
||||
from forja_core.llm.mock import MockProvider
|
||||
|
||||
|
||||
@@ -18,3 +22,46 @@ def test_factory_azure_requiere_credenciales() -> None:
|
||||
s = Settings(llm_provider="azure", _env_file=None) # type: ignore[call-arg]
|
||||
with pytest.raises(ValueError):
|
||||
build_llm_provider(s)
|
||||
|
||||
|
||||
def test_factory_envuelve_con_fallback_cuando_difiere() -> None:
|
||||
s = Settings(
|
||||
llm_provider="openai",
|
||||
llm_fallback_provider="mock",
|
||||
openai_api_key="sk-test",
|
||||
_env_file=None,
|
||||
) # type: ignore[call-arg]
|
||||
p = build_llm_provider(s)
|
||||
assert isinstance(p, FallbackLLMProvider)
|
||||
assert p.name == "openai+fallback:mock"
|
||||
|
||||
|
||||
def test_factory_ignora_fallback_igual_al_primario() -> None:
|
||||
s = Settings(llm_provider="mock", llm_fallback_provider="mock", _env_file=None) # type: ignore[call-arg]
|
||||
p = build_llm_provider(s)
|
||||
assert isinstance(p, MockProvider)
|
||||
|
||||
|
||||
class _FailingProvider:
|
||||
name = "failing"
|
||||
|
||||
async def complete(
|
||||
self,
|
||||
messages: list[Message],
|
||||
schema: dict[str, Any] | None = None,
|
||||
temperature: float = 0.2,
|
||||
max_tokens: int = 2000,
|
||||
) -> CompletionResult:
|
||||
raise RuntimeError("primary down")
|
||||
|
||||
|
||||
async def test_fallback_delega_cuando_el_primario_falla() -> None:
|
||||
provider = FallbackLLMProvider(_FailingProvider(), MockProvider())
|
||||
result = await provider.complete([Message(role="user", content="mos degradation")])
|
||||
assert result.model.startswith("mock")
|
||||
|
||||
|
||||
async def test_fallback_no_interviene_si_el_primario_responde() -> None:
|
||||
provider = FallbackLLMProvider(MockProvider(), _FailingProvider())
|
||||
result = await provider.complete([Message(role="user", content="mos degradation")])
|
||||
assert result.model.startswith("mock")
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Tests for dataset and model YAML registries."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from forja_core.registry.dataset_store import FileSystemDatasetStore
|
||||
from forja_core.registry.model_store import FileSystemModelStore
|
||||
|
||||
FIXTURES = Path(__file__).parent.parent / "fixtures"
|
||||
|
||||
|
||||
def test_get_dataset_active_version() -> None:
|
||||
store = FileSystemDatasetStore(FIXTURES / "datasets")
|
||||
ds = store.get_dataset("incident_sft")
|
||||
assert ds.name == "incident_sft"
|
||||
assert ds.version == "v1"
|
||||
assert ds.format == "jsonl"
|
||||
|
||||
|
||||
def test_get_model_active_version() -> None:
|
||||
store = FileSystemModelStore(FIXTURES / "models")
|
||||
m = store.get_model("gpt4o_lora_base")
|
||||
assert m.adaptation == "lora"
|
||||
assert m.base_model_id == "gpt-4o"
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Post-training evaluation over canonical scenarios."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from forja_core.api.deps import (
|
||||
get_guardrail_engine,
|
||||
get_llm_provider,
|
||||
get_orchestrator,
|
||||
get_policy_store,
|
||||
get_registry,
|
||||
get_settings,
|
||||
)
|
||||
from forja_core.evaluation.post_train import run_post_train_evaluation
|
||||
from forja_core.evaluation.scenarios import load_agent_scenarios
|
||||
|
||||
FIXTURES = Path(__file__).parent.parent / "fixtures"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def patch_settings(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
||||
monkeypatch.setenv("AGENTS_DIR", str(FIXTURES / "agents"))
|
||||
monkeypatch.setenv("POLICIES_DIR", str(FIXTURES / "policies"))
|
||||
for fn in (
|
||||
get_settings,
|
||||
get_registry,
|
||||
get_policy_store,
|
||||
get_orchestrator,
|
||||
get_llm_provider,
|
||||
get_guardrail_engine,
|
||||
):
|
||||
fn.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_scenarios_from_examples() -> None:
|
||||
scenarios = load_agent_scenarios(FIXTURES / "agents", "test_agent")
|
||||
assert len(scenarios) >= 1
|
||||
assert scenarios[0][0].endswith(".txt")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_train_eval_passes_for_test_agent() -> None:
|
||||
registry = get_registry()
|
||||
policies = get_policy_store()
|
||||
orchestrator = get_orchestrator()
|
||||
settings = get_settings()
|
||||
agent = registry.get_agent("test_agent")
|
||||
policy = policies.get_policy(agent.guardrails[0])
|
||||
from uuid import uuid4
|
||||
|
||||
report = await run_post_train_evaluation(
|
||||
training_run_id=uuid4(),
|
||||
agent_def=agent,
|
||||
policy=policy,
|
||||
orchestrator=orchestrator,
|
||||
agents_dir=settings.agents_dir,
|
||||
)
|
||||
assert report.scenario_count >= 1
|
||||
assert report.passed is True
|
||||
assert report.violation_count == 0
|
||||
@@ -20,8 +20,8 @@ def test_get_agent_carga_version_activa() -> None:
|
||||
def test_list_versions_devuelve_metadata() -> None:
|
||||
r = FileSystemAgentRegistry(FIXTURES)
|
||||
metas = r.list_versions("test_agent")
|
||||
assert len(metas) == 1
|
||||
assert metas[0].id == "v1"
|
||||
assert len(metas) == 2
|
||||
assert {m.id for m in metas} == {"v1", "v2"}
|
||||
|
||||
|
||||
def test_upsert_crea_nueva_version_y_actualiza_index(tmp_path: Path) -> None:
|
||||
|
||||
@@ -106,7 +106,10 @@ async def test_grafo_bloquea_por_guardrail_de_entrada(tmp_path: Path) -> None:
|
||||
version="v1",
|
||||
description="bloquea emails en la entrada",
|
||||
input_validators=[
|
||||
{"type": "detect_pii", "config": {"entities": ["EMAIL_ADDRESS"], "severity_on_match": "block"}}
|
||||
{
|
||||
"type": "detect_pii",
|
||||
"config": {"entities": ["EMAIL_ADDRESS"], "severity_on_match": "block"},
|
||||
}
|
||||
],
|
||||
output_validators=[],
|
||||
)
|
||||
|
||||
@@ -79,7 +79,9 @@ async def test_snapshot_relee_estado_sin_avanzar(orchestrator: AgentOrchestrator
|
||||
|
||||
|
||||
async def test_snapshot_trace_id_desconocido_devuelve_none(orchestrator: AgentOrchestrator) -> None:
|
||||
assert await orchestrator.snapshot(agent_def=_agent(), policy=_policy(), trace_id=uuid4()) is None
|
||||
assert (
|
||||
await orchestrator.snapshot(agent_def=_agent(), policy=_policy(), trace_id=uuid4()) is None
|
||||
)
|
||||
|
||||
|
||||
async def test_invoke_pausa_hitl_y_resume_aprueba(orchestrator: AgentOrchestrator) -> None:
|
||||
@@ -137,16 +139,12 @@ async def test_invoke_bloqueado_por_guardrail(orchestrator: AgentOrchestrator) -
|
||||
|
||||
async def test_estado_persiste_entre_instancias(tmp_path: Path) -> None:
|
||||
agent = _agent()
|
||||
o1 = AgentOrchestrator(
|
||||
provider=MockProvider(), engine=GuardrailsAIEngine(), data_dir=tmp_path
|
||||
)
|
||||
o1 = AgentOrchestrator(provider=MockProvider(), engine=GuardrailsAIEngine(), data_dir=tmp_path)
|
||||
paused = await o1.invoke(agent_def=agent, policy=_policy(), user_input="caída registros sip")
|
||||
assert paused.status == "awaiting_approval"
|
||||
|
||||
# Nueva instancia (simula reinicio del proceso) apuntando al mismo data_dir.
|
||||
o2 = AgentOrchestrator(
|
||||
provider=MockProvider(), engine=GuardrailsAIEngine(), data_dir=tmp_path
|
||||
)
|
||||
o2 = AgentOrchestrator(provider=MockProvider(), engine=GuardrailsAIEngine(), data_dir=tmp_path)
|
||||
resumed = await o2.resume(
|
||||
agent_def=agent,
|
||||
policy=_policy(),
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Tests for mock and Azure ML stub training backends."""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from forja_core.config import Settings
|
||||
from forja_core.domain.training import TrainingHyperparameters, TrainingJobSpec
|
||||
from forja_core.training.azure_ml import AzureMLTrainingBackend
|
||||
from forja_core.training.mock import MockTrainingBackend
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def spec() -> TrainingJobSpec:
|
||||
return TrainingJobSpec(
|
||||
dataset_name="incident_sft",
|
||||
dataset_version="v1",
|
||||
model_name="gpt4o_lora_base",
|
||||
model_version="v1",
|
||||
hyperparameters=TrainingHyperparameters(epochs=2),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mock_submit_and_status_succeeds(spec: TrainingJobSpec) -> None:
|
||||
backend = MockTrainingBackend()
|
||||
run_id = uuid4()
|
||||
job_id = await backend.submit(spec, run_id)
|
||||
assert job_id == f"mock:{run_id}"
|
||||
status = await backend.status(job_id)
|
||||
assert status.status == "succeeded"
|
||||
assert len(status.artifacts) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_azure_stub_submit_and_status_succeeds(spec: TrainingJobSpec) -> None:
|
||||
backend = AzureMLTrainingBackend(settings=Settings())
|
||||
run_id = uuid4()
|
||||
job_id = await backend.submit(spec, run_id)
|
||||
assert job_id.startswith("azureml-stub:")
|
||||
status = await backend.status(job_id)
|
||||
assert status.status == "succeeded"
|
||||
assert "azureml://" in status.artifacts[0].uri
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Single-page UI tests: stages render, fragments refresh, and the full
|
||||
click-through lifecycle (run → approve → train → evaluate → promote) works."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from forja_core.api import deps
|
||||
from forja_core.main import create_app
|
||||
|
||||
FIXTURES = Path(__file__).parent.parent / "fixtures"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def patch_settings(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import shutil
|
||||
|
||||
# Agents are copied to tmp because the promotion test mutates the registry.
|
||||
agents_dir = tmp_path / "agents"
|
||||
shutil.copytree(FIXTURES / "agents", agents_dir)
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path / "data"))
|
||||
monkeypatch.setenv("AGENTS_DIR", str(agents_dir))
|
||||
monkeypatch.setenv("POLICIES_DIR", str(FIXTURES / "policies"))
|
||||
monkeypatch.setenv("DATASETS_DIR", str(FIXTURES / "datasets"))
|
||||
monkeypatch.setenv("MODELS_DIR", str(FIXTURES / "models"))
|
||||
monkeypatch.setenv("TRAINING_BACKEND", "mock")
|
||||
monkeypatch.setenv("LLM_PROVIDER", "mock")
|
||||
for fn in (
|
||||
deps.get_settings,
|
||||
deps.get_registry,
|
||||
deps.get_policy_store,
|
||||
deps.get_dataset_store,
|
||||
deps.get_model_store,
|
||||
deps.get_training_backend,
|
||||
deps.get_orchestrator,
|
||||
deps.get_llm_provider,
|
||||
deps.get_guardrail_engine,
|
||||
):
|
||||
fn.cache_clear()
|
||||
|
||||
|
||||
def test_index_renders_all_six_stages() -> None:
|
||||
r = TestClient(create_app()).get("/")
|
||||
assert r.status_code == 200
|
||||
for anchor in (
|
||||
'id="define"',
|
||||
'id="run"',
|
||||
'id="approve"',
|
||||
'id="train"',
|
||||
'id="promote"',
|
||||
'id="audit"',
|
||||
):
|
||||
assert anchor in r.text
|
||||
# Define stage: agent with version/state chips and the policy validators.
|
||||
assert "test_agent" in r.text
|
||||
assert "v1 · active" in r.text
|
||||
assert "v2 · draft" in r.text
|
||||
assert "detect_pii" in r.text
|
||||
# Run stage: the one-click demo button.
|
||||
assert "Run demo incident" in r.text
|
||||
|
||||
|
||||
def test_run_invoke_completes_and_lands_in_history_fragment() -> None:
|
||||
client = TestClient(create_app())
|
||||
# "mos" maps to a low-risk canonical mock response → completes without HITL.
|
||||
r = client.post("/run", data={"agent_name": "test_agent", "input": "mos degradation in pool"})
|
||||
assert r.status_code == 200
|
||||
assert "Completed" in r.text
|
||||
assert "validate_input" in r.text
|
||||
assert "finalize" in r.text
|
||||
|
||||
h = client.get("/fragments/history")
|
||||
assert h.status_code == 200
|
||||
assert "test_agent" in h.text
|
||||
|
||||
|
||||
def test_run_invoke_hitl_appears_in_approvals_fragment() -> None:
|
||||
client = TestClient(create_app())
|
||||
# "hss" maps to a risk-5 action requiring approval → execution pauses in HITL.
|
||||
r = client.post("/run", data={"agent_name": "test_agent", "input": "hss capacity alarm"})
|
||||
assert r.status_code == 200
|
||||
assert "Pending approval" in r.text
|
||||
|
||||
a = client.get("/fragments/approvals")
|
||||
assert a.status_code == 200
|
||||
assert "test_agent" in a.text
|
||||
assert "isolate_replication_link" in a.text
|
||||
assert "Approve all" in a.text
|
||||
|
||||
|
||||
def test_run_invoke_unknown_agent_returns_error() -> None:
|
||||
client = TestClient(create_app())
|
||||
r = client.post("/run", data={"agent_name": "missing", "input": "x"})
|
||||
assert r.status_code == 400
|
||||
assert "Execution error" in r.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("path", "target"),
|
||||
[
|
||||
("/agents", "/#define"),
|
||||
("/policies", "/#define"),
|
||||
("/run", "/#run"),
|
||||
("/approvals", "/#approve"),
|
||||
("/training", "/#train"),
|
||||
("/promotions", "/#promote"),
|
||||
("/history", "/#audit"),
|
||||
("/armory", "/#audit"),
|
||||
("/agents/test_agent", "/#define"),
|
||||
],
|
||||
)
|
||||
def test_legacy_routes_redirect_to_anchors(path: str, target: str) -> None:
|
||||
client = TestClient(create_app())
|
||||
r = client.get(path, follow_redirects=False)
|
||||
assert r.status_code == 301
|
||||
assert r.headers["location"] == target
|
||||
|
||||
|
||||
def test_full_lifecycle_click_through() -> None:
|
||||
"""The exact sequence of requests the page buttons fire, end to end."""
|
||||
client = TestClient(create_app())
|
||||
|
||||
# Stage 02 click: run the demo incident → pauses at the approval gate.
|
||||
r = client.post("/run", data={"agent_name": "test_agent", "input": "hss capacity alarm"})
|
||||
assert "Pending approval" in r.text
|
||||
|
||||
# Stage 03 click: approve all pending actions → run completes.
|
||||
pending = client.get("/api/executions").json()
|
||||
trace_id = next(e["trace_id"] for e in pending if e["status"] == "awaiting_approval")
|
||||
approved = client.post(
|
||||
f"/api/executions/{trace_id}/approve",
|
||||
json={"approved_action_ids": [], "comment": "Approved from UI"},
|
||||
).json()
|
||||
assert approved["status"] == "completed"
|
||||
assert client.get("/fragments/approvals").text.count("Pending approval") == 0
|
||||
assert "test_agent" in client.get("/fragments/history").text
|
||||
|
||||
# Stage 04 click: submit a training run for the draft candidate (v2).
|
||||
run = client.post(
|
||||
"/api/training/runs",
|
||||
json={
|
||||
"dataset_name": "incident_sft",
|
||||
"model_name": "gpt4o_lora_base",
|
||||
"agent_name": "test_agent",
|
||||
"agent_version": "v2",
|
||||
},
|
||||
).json()
|
||||
assert run["status"] == "succeeded"
|
||||
frag = client.get("/fragments/training").text
|
||||
assert "Run evaluation" in frag
|
||||
|
||||
# Stage 04 click: evaluate the candidate.
|
||||
report = client.post(f"/api/training/runs/{run['id']}/evaluate").json()
|
||||
assert report["passed"] is True
|
||||
frag = client.get("/fragments/training").text
|
||||
assert "Request promotion" in frag
|
||||
|
||||
# Stage 04 click: request promotion → shows as pending in stages 04 and 05.
|
||||
promo = client.post(
|
||||
"/api/promotions",
|
||||
json={
|
||||
"agent_name": "test_agent",
|
||||
"agent_version": "v2",
|
||||
"training_run_id": run["id"],
|
||||
"evaluation_report_id": report["id"],
|
||||
"requested_by": "web-ui",
|
||||
},
|
||||
)
|
||||
assert promo.status_code == 201
|
||||
assert "Promotion pending" in client.get("/fragments/training").text
|
||||
assert "test_agent" in client.get("/fragments/promotions").text
|
||||
|
||||
# Stage 05 click: approve the promotion → candidate flips to active.
|
||||
approve = client.post(
|
||||
f"/api/promotions/{promo.json()['id']}/approve",
|
||||
json={"approved_by": "web-ui", "comment": "ok"},
|
||||
)
|
||||
assert approve.status_code == 200
|
||||
assert "✓ Promoted" in client.get("/fragments/training").text
|
||||
assert "v2 · active" in client.get("/fragments/agents").text
|
||||
|
||||
agent = client.get("/api/agents/test_agent").json()
|
||||
assert agent["version"] == "v2"
|
||||
assert agent["state"] == "active"
|
||||
Reference in New Issue
Block a user