From 37633920ceb75e495ab63e46483321d57f2939ac Mon Sep 17 00:00:00 2001 From: Juan Marquez Date: Wed, 10 Jun 2026 18:21:33 +0200 Subject: [PATCH] pagina unica --- .env.example | 21 +- .gitignore | 5 +- ARCHITECTURE.md | 20 +- README.md | 53 ++- agents/incident_analyzer/index.yaml | 5 + agents/incident_analyzer/versions/v2.yaml | 2 + agents/incident_analyzer/versions/v3.yaml | 43 ++ common/src/forja_common/__init__.py | 5 - common/src/forja_common/client.py | 87 ---- core/requirements.txt | 5 +- core/src/forja_core/api/agents.py | 9 +- core/src/forja_core/api/datasets.py | 39 ++ core/src/forja_core/api/deps.py | 29 +- .../forja_core/api/evaluation_persistence.py | 47 ++ core/src/forja_core/api/executions.py | 39 +- core/src/forja_core/api/middlewares.py | 4 +- core/src/forja_core/api/models.py | 39 ++ core/src/forja_core/api/policies.py | 16 +- .../forja_core/api/promotion_persistence.py | 51 +++ core/src/forja_core/api/promotions.py | 202 +++++++++ core/src/forja_core/api/training.py | 237 ++++++++++ .../forja_core/api/training_persistence.py | 42 ++ core/src/forja_core/config.py | 20 +- core/src/forja_core/domain/agent.py | 12 +- core/src/forja_core/domain/dataset.py | 28 ++ .../src/forja_core/domain/foundation_model.py | 28 ++ core/src/forja_core/domain/policy.py | 13 +- core/src/forja_core/domain/training.py | 127 ++++++ core/src/forja_core/domain/version_meta.py | 17 + core/src/forja_core/evaluation/__init__.py | 1 + core/src/forja_core/evaluation/post_train.py | 85 ++++ core/src/forja_core/evaluation/scenarios.py | 16 + core/src/forja_core/guardrails/factory.py | 27 +- core/src/forja_core/guardrails/nemo.py | 2 +- core/src/forja_core/guardrails/validators.py | 16 +- core/src/forja_core/llm/factory.py | 18 +- core/src/forja_core/llm/fallback.py | 46 ++ core/src/forja_core/main.py | 15 +- core/src/forja_core/registry/dataset_store.py | 25 ++ core/src/forja_core/registry/factory.py | 10 + core/src/forja_core/registry/model_store.py | 25 ++ core/src/forja_core/registry/policy_store.py | 96 +--- core/src/forja_core/registry/repository.py | 113 ++--- core/src/forja_core/registry/yaml_store.py | 133 ++++++ core/src/forja_core/runtime/orchestrator.py | 4 +- core/src/forja_core/training/__init__.py | 1 + .../training/_azure_job_bundle/train.py | 25 ++ core/src/forja_core/training/azure_ml.py | 67 +++ .../forja_core/training/azure_ml_config.py | 25 ++ core/src/forja_core/training/azure_ml_sdk.py | 100 +++++ core/src/forja_core/training/base.py | 22 + core/src/forja_core/training/factory.py | 14 + core/src/forja_core/training/mock.py | 28 ++ .../src/forja_core/web/templates/_agents.html | 79 ++++ .../forja_core/web/templates/_approvals.html | 53 +++ .../forja_core/web/templates/_history.html | 27 ++ .../forja_core/web/templates/_promotions.html | 61 +++ .../forja_core/web/templates/_training.html | 98 +++++ .../web/templates/agent_detail.html | 103 ----- .../web/templates/agent_detail_full.html | 11 - core/src/forja_core/web/templates/agents.html | 50 --- .../forja_core/web/templates/approvals.html | 51 --- core/src/forja_core/web/templates/armory.html | 61 --- core/src/forja_core/web/templates/base.html | 137 +++--- core/src/forja_core/web/templates/deploy.html | 48 -- core/src/forja_core/web/templates/home.html | 40 -- core/src/forja_core/web/templates/index.html | 223 ++++++++++ core/src/forja_core/web/templates/run.html | 38 -- .../forja_core/web/templates/run_result.html | 68 +++ .../forja_core/web/templates/tutorial.html | 90 ---- core/src/forja_core/web/ui.py | 411 ++++++++++-------- datasets/incident_sft/index.yaml | 8 + datasets/incident_sft/records/v1.jsonl | 1 + datasets/incident_sft/versions/v1.yaml | 8 + docker-compose.yml | 6 + docs/componentes.md | 32 +- docs/explicacion.md | 106 ++--- docs/forja-progress.html | 124 ------ docs/futuro.md | 25 +- docs/product-voice.md | 51 +++ docs/project-progress.html | 220 ++++++++++ models/gpt4o_lora_base/index.yaml | 8 + models/gpt4o_lora_base/versions/v1.yaml | 8 + pyproject.toml | 17 +- resume.txt | 1 + .../agents/test_agent/examples/basic.txt | 1 + tests/fixtures/agents/test_agent/index.yaml | 9 +- .../agents/test_agent/versions/v2.yaml | 18 + .../fixtures/datasets/incident_sft/index.yaml | 8 + .../datasets/incident_sft/versions/v1.yaml | 8 + .../models/gpt4o_lora_base/index.yaml | 8 + .../models/gpt4o_lora_base/versions/v1.yaml | 8 + tests/integration/test_invoke_happy_path.py | 4 +- tests/integration/test_invoke_hitl.py | 4 +- .../test_invoke_resume_after_restart.py | 4 +- tests/unit/test_api_executions.py | 4 +- tests/unit/test_api_policies_violations.py | 4 +- tests/unit/test_api_promotions.py | 173 ++++++++ tests/unit/test_api_training.py | 70 +++ tests/unit/test_azure_ml_sdk.py | 96 ++++ tests/unit/test_llm_factory.py | 47 ++ tests/unit/test_ml_registry.py | 23 + tests/unit/test_post_train_eval.py | 63 +++ tests/unit/test_registry.py | 4 +- tests/unit/test_runtime_graph.py | 5 +- tests/unit/test_runtime_orchestrator.py | 12 +- tests/unit/test_training_backend.py | 43 ++ tests/unit/test_ui_pages.py | 185 ++++++++ 108 files changed, 3874 insertions(+), 1350 deletions(-) create mode 100644 agents/incident_analyzer/versions/v3.yaml delete mode 100644 common/src/forja_common/__init__.py delete mode 100644 common/src/forja_common/client.py create mode 100644 core/src/forja_core/api/datasets.py create mode 100644 core/src/forja_core/api/evaluation_persistence.py create mode 100644 core/src/forja_core/api/models.py create mode 100644 core/src/forja_core/api/promotion_persistence.py create mode 100644 core/src/forja_core/api/promotions.py create mode 100644 core/src/forja_core/api/training.py create mode 100644 core/src/forja_core/api/training_persistence.py create mode 100644 core/src/forja_core/domain/dataset.py create mode 100644 core/src/forja_core/domain/foundation_model.py create mode 100644 core/src/forja_core/domain/training.py create mode 100644 core/src/forja_core/domain/version_meta.py create mode 100644 core/src/forja_core/evaluation/__init__.py create mode 100644 core/src/forja_core/evaluation/post_train.py create mode 100644 core/src/forja_core/evaluation/scenarios.py create mode 100644 core/src/forja_core/llm/fallback.py create mode 100644 core/src/forja_core/registry/dataset_store.py create mode 100644 core/src/forja_core/registry/model_store.py create mode 100644 core/src/forja_core/registry/yaml_store.py create mode 100644 core/src/forja_core/training/__init__.py create mode 100644 core/src/forja_core/training/_azure_job_bundle/train.py create mode 100644 core/src/forja_core/training/azure_ml.py create mode 100644 core/src/forja_core/training/azure_ml_config.py create mode 100644 core/src/forja_core/training/azure_ml_sdk.py create mode 100644 core/src/forja_core/training/base.py create mode 100644 core/src/forja_core/training/factory.py create mode 100644 core/src/forja_core/training/mock.py create mode 100644 core/src/forja_core/web/templates/_agents.html create mode 100644 core/src/forja_core/web/templates/_approvals.html create mode 100644 core/src/forja_core/web/templates/_history.html create mode 100644 core/src/forja_core/web/templates/_promotions.html create mode 100644 core/src/forja_core/web/templates/_training.html delete mode 100644 core/src/forja_core/web/templates/agent_detail.html delete mode 100644 core/src/forja_core/web/templates/agent_detail_full.html delete mode 100644 core/src/forja_core/web/templates/agents.html delete mode 100644 core/src/forja_core/web/templates/approvals.html delete mode 100644 core/src/forja_core/web/templates/armory.html delete mode 100644 core/src/forja_core/web/templates/deploy.html delete mode 100644 core/src/forja_core/web/templates/home.html create mode 100644 core/src/forja_core/web/templates/index.html delete mode 100644 core/src/forja_core/web/templates/run.html create mode 100644 core/src/forja_core/web/templates/run_result.html delete mode 100644 core/src/forja_core/web/templates/tutorial.html create mode 100644 datasets/incident_sft/index.yaml create mode 100644 datasets/incident_sft/records/v1.jsonl create mode 100644 datasets/incident_sft/versions/v1.yaml delete mode 100644 docs/forja-progress.html create mode 100644 docs/product-voice.md create mode 100644 docs/project-progress.html create mode 100644 models/gpt4o_lora_base/index.yaml create mode 100644 models/gpt4o_lora_base/versions/v1.yaml create mode 100644 resume.txt create mode 100644 tests/fixtures/agents/test_agent/examples/basic.txt create mode 100644 tests/fixtures/agents/test_agent/versions/v2.yaml create mode 100644 tests/fixtures/datasets/incident_sft/index.yaml create mode 100644 tests/fixtures/datasets/incident_sft/versions/v1.yaml create mode 100644 tests/fixtures/models/gpt4o_lora_base/index.yaml create mode 100644 tests/fixtures/models/gpt4o_lora_base/versions/v1.yaml create mode 100644 tests/unit/test_api_promotions.py create mode 100644 tests/unit/test_api_training.py create mode 100644 tests/unit/test_azure_ml_sdk.py create mode 100644 tests/unit/test_ml_registry.py create mode 100644 tests/unit/test_post_train_eval.py create mode 100644 tests/unit/test_training_backend.py create mode 100644 tests/unit/test_ui_pages.py diff --git a/.env.example b/.env.example index 9049403..c6250fe 100644 --- a/.env.example +++ b/.env.example @@ -16,12 +16,27 @@ OPENAI_MODEL=gpt-4o # Guardrails GUARDRAILS_NEMO_ENABLED=false +# Keywords CSV para topical rails de NeMo (vacío = sin chequeo off-topic) +GUARDRAILS_NEMO_ALLOWED_KEYWORDS= # Observabilidad LOG_LEVEL=INFO +# Training orchestration (mock | azure_ml) +TRAINING_BACKEND=mock + +# Azure ML (required when TRAINING_BACKEND=azure_ml and using live SDK; else stub mode) +AZURE_ML_SUBSCRIPTION_ID= +AZURE_ML_RESOURCE_GROUP= +AZURE_ML_WORKSPACE_NAME= +AZURE_ML_COMPUTE=cpu-cluster +AZURE_ML_ENVIRONMENT=AzureML-sklearn-1.5 +AZURE_ML_EXPERIMENT_NAME=forja-training +AZURE_ML_COMMAND=python train.py + # Persistencia DATA_DIR=./data - -# URL del core (la usa el dashboard) -FORJA_CORE_URL=http://core:8000 +AGENTS_DIR=./agents +POLICIES_DIR=./policies +DATASETS_DIR=./datasets +MODELS_DIR=./models diff --git a/.gitignore b/.gitignore index a88e8fb..1629e5b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,11 +4,12 @@ *.key *.pem -# Datos runtime (registry, checkpoints, logs) +# Datos runtime (registry, checkpoints, logs). +# Nota: sin patrón global *.jsonl — los records de datasets/ (p. ej. +# datasets//records/*.jsonl) son contenido versionado, no runtime. data/ *.sqlite *.sqlite-* -*.jsonl # Python __pycache__/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 71b8b71..4433fc5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -35,11 +35,24 @@ Forja es un **único servicio FastAPI** (:8000): | Capa | Tecnología | Por qué | |---|---|---| -| Agent Registry | JSON + YAML | Humano lo escribe (YAML); máquina lo cataloga (index.yaml). | -| Versionado | YAML por versión + index.yaml | Diff legible; hash SHA-256 normalizado. | +| Registries (agents/policies/datasets/models) | YAML por versión + index.yaml | Humano lo escribe; diff legible; hash SHA-256 normalizado. Base común en `registry/yaml_store.py`. | | Logs de violaciones | JSONL append-only | Inmutable, auditable, fácil de shipear. | | Logs de ejecuciones | JSONL append-only | Idem. | | Checkpoints LangGraph | SQLite | Tool right; persistencia entre restarts (HITL). | +| Training runs / evaluaciones / promociones | JSON por entidad bajo `data/` | Estado mutable consultable por id; los registros de promoción aprobada van además a JSONL append-only. | + +## Ciclo MLOps (training → evaluación → promoción) + +1. `POST /api/training/runs` envía un job al backend configurado + (`TRAINING_BACKEND=mock|azure_ml`) con dataset, modelo base y, opcionalmente, + el agente y su **versión candidata**. +2. `POST /api/training/runs/{id}/evaluate` ejecuta los escenarios canónicos + (`agents//examples/*.txt`) contra la versión candidata por el runtime + gobernado completo; pasa si no hay violaciones bloqueantes ni fallos. +3. `POST /api/promotions` crea la petición; exige run `succeeded`, evaluación + `passed` y que la evaluación cubra **exactamente** la versión a promocionar. +4. Un operador aprueba/rechaza (UI `/promotions`); al aprobar, la versión pasa a + `active` en el registry y queda un registro de auditoría append-only. ## HITL — Patrón @@ -57,7 +70,8 @@ nodo `approve_gate`. Sólo pausa cuando hay acciones de alto riesgo ## Errores -- Azure OpenAI / OpenAI: retry exponencial 1s/2s/4s, fallback opcional, luego +- Azure OpenAI / OpenAI: retry exponencial 1s/2s/4s; si `LLM_FALLBACK_PROVIDER` + está configurado, `FallbackLLMProvider` delega en el secundario; luego `status=failed`. - Validador: `fail_closed` por defecto. - Schema mismatch en LLM output: 1 retry, luego `failed`. diff --git a/README.md b/README.md index 7074dc5..8aff4cf 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,11 @@ ejecución controlada con runtime stateful, Human-in-the-Loop y observabilidad c Forja es un **único servicio** (FastAPI en :8000) que expone: -- API REST completa bajo `/api` (agentes, políticas, ejecuciones, aprobaciones, violaciones). -- UI embebida moderna (HTMX + Tailwind + Jinja) en rutas amigables (`/agents`, `/run`, `/approvals`, `/history`, `/policies`). +- API REST completa bajo `/api` (agentes, políticas, ejecuciones, aprobaciones, violaciones, datasets, models, training runs, promotions). +- UI embebida de **una sola página** (HTMX + Tailwind + Jinja) que recorre el + ciclo de vida completo en seis etapas: **Define → Run → Approve → Train → + Promote → Audit**. Cada panel se refresca solo tras cada acción; las rutas + antiguas (`/agents`, `/run`, ...) redirigen a su etapa. ``` ┌───────────────────────────────────────────────────────────────┐ @@ -42,26 +45,39 @@ Abre [http://localhost:8000](http://localhost:8000). Funciona out-of-the-box (`LLM_PROVIDER=mock`, sin API keys). Si quieres usar Azure OpenAI real, edita `.env`. -## Demo guiada (3 pasos) +## Demo guiada: el ciclo completo en 6 clicks -1. **Registro** → ver `incident_analyzer` y comparar `v1` vs `v2`. -2. **Ejecutar** → seleccionar el escenario `01_sip_registration_drop` y - pulsar *Invocar*. Verás el grafo recorrer validate_input → llm_reason - → validate_output → propose_actions → approve_gate y pausarse en HITL. -3. **Aprobaciones** → revisar las acciones propuestas (con risk_score y - rollback_plan), aprobar las seguras y comprobar que la ejecución - completa. +Todo ocurre en `http://localhost:8000`, de arriba abajo, sin escribir nada: + +1. **▶ Run demo incident** (etapa 02) — lanza un incidente HSS pregrabado por el + pipeline gobernado. La acción propuesta tiene risk 5 → la ejecución se pausa + en el approval gate. +2. **Approve all** (etapa 03) — el grafo se reanuda desde su checkpoint y la + ejecución completa (aparece en la etapa 06 — Audit). +3. **Submit training run** (etapa 04) — el formulario viene pre-rellenado con el + dataset `incident_sft`, el modelo `gpt4o_lora_base` y la versión candidata + `v3` (draft). Con `TRAINING_BACKEND=mock` termina al instante. +4. **Run evaluation** (etapa 04) — ejecuta los escenarios canónicos del agente + (`agents//examples/`) sobre la candidata `v3`; pasa si no hay + violaciones bloqueantes. +5. **Request promotion** (etapa 04) — crea la petición de gobernanza, que + aparece en la etapa 05. +6. **Approve promotion** (etapa 05) — `v3` pasa a `active` en el registry; + compruébalo en los chips de versión de la etapa 01. ## Capacidades implementadas | Feature | Ubicación | |---|---| -| Agent / Policy Registry | `core/src/forja_core/registry/` | -| Versionado Git-like (YAML) | `agents//` + `policies//` | +| Agent / Policy / Dataset / Model Registry | `core/src/forja_core/registry/` (base genérica `yaml_store.py`) | +| Versionado Git-like (YAML) | `agents/`, `policies/`, `datasets/`, `models/` | | Guardrails runtime (strategy) | `core/src/forja_core/guardrails/` | | LangGraph + HITL + checkpoints | `core/src/forja_core/runtime/` | | Observabilidad (trace + structlog) | `core/src/forja_core/observability/` | -| LLM abstraction | `core/src/forja_core/llm/` | +| LLM abstraction (+fallback opcional) | `core/src/forja_core/llm/` | +| Training backends (mock / Azure ML) | `core/src/forja_core/training/` | +| Evaluación post-training (escenarios) | `core/src/forja_core/evaluation/` | +| Promoción gobernada draft → active | `core/src/forja_core/api/promotions.py` | | UI embebida (HTMX) | `core/src/forja_core/web/` (ruta raíz) | | API REST completa | bajo `/api` en forja-core :8000 | @@ -69,6 +85,10 @@ Azure OpenAI real, edita `.env`. Ver [`.env.example`](.env.example). +### Azure ML training + +Set `TRAINING_BACKEND=azure_ml` and fill `AZURE_ML_*` workspace settings. Forja submits an Azure ML **command job** via `azure-ai-ml` using `DefaultAzureCredential` (Azure CLI, managed identity, etc.). Replace the bundled script in `core/src/forja_core/training/_azure_job_bundle/` or override `AZURE_ML_COMMAND` for your fine-tuning pipeline. Without workspace config, the backend stays in **stub mode** for local development. + ## Roadmap Ver [`docs/futuro.md`](docs/futuro.md). @@ -80,11 +100,18 @@ forja/ ├── core/ servicio FastAPI único (API + UI HTMX embebida) ├── agents/ definiciones de agentes (YAML versionados) ├── policies/ políticas de guardrails (YAML versionados) +├── datasets/ datasets para fine-tuning (YAML versionados) +├── models/ modelos base / destino de entrenamiento (YAML versionados) ├── data/ runtime state (gitignored) ├── tests/ └── docs/ (explicacion.md, componentes.md, futuro.md, ARCHITECTURE.md) ``` +## Development + +- Implementation guidelines: [`karpathy.md`](karpathy.md) +- UI copy and language: [`docs/product-voice.md`](docs/product-voice.md) (English only in the web app) + ## Tests ```bash diff --git a/agents/incident_analyzer/index.yaml b/agents/incident_analyzer/index.yaml index 4f48238..3f48671 100644 --- a/agents/incident_analyzer/index.yaml +++ b/agents/incident_analyzer/index.yaml @@ -10,4 +10,9 @@ versions: author: Juan message: Añade detección de codec mismatch y refuerzo de prompts. created_at: 2026-05-01T12:00:00Z + - id: v3 + hash: pending + author: Juan + message: Candidata SFT sobre incident_sft (LoRA); pendiente de evaluación y promoción. + created_at: 2026-06-10T10:00:00Z active_version: v2 diff --git a/agents/incident_analyzer/versions/v2.yaml b/agents/incident_analyzer/versions/v2.yaml index 3dedecd..d3726c3 100644 --- a/agents/incident_analyzer/versions/v2.yaml +++ b/agents/incident_analyzer/versions/v2.yaml @@ -35,6 +35,8 @@ output_schema: type: object required: [id, action, target, risk_score, rollback_plan, requires_approval] risk_threshold_for_hitl: 4 +input_label: Incident description +input_placeholder: "Sudden 80% drop in SIP registrations after deploying image 4.7.2 on the CSCF cluster." icon: "🤖" color: "#f59e0b" updated_at: 2026-05-01T12:00:00Z diff --git a/agents/incident_analyzer/versions/v3.yaml b/agents/incident_analyzer/versions/v3.yaml new file mode 100644 index 0000000..5afd5ed --- /dev/null +++ b/agents/incident_analyzer/versions/v3.yaml @@ -0,0 +1,43 @@ +name: incident_analyzer +version: v3 +owner: Juan +purpose: | + Candidata re-entrenada (SFT sobre incident_sft, LoRA) con la misma cobertura + que v2: codec mismatch, tormentas de registro post-handover, degradación de + MOS y saturación replicada en HSS active-active. +state: draft +guardrails: [default] +llm: + provider: mock + model: gpt-4o + temperature: 0.1 + max_tokens: 2000 +system_prompt: | + Eres un analista senior de operaciones de plataforma de voz virtualizada (IMS, + CSCF, SBC, HSS). Recibes una descripción de incidente. Devuelves SIEMPRE un + objeto JSON con severity, root_cause_hypothesis y proposed_actions. + Considera específicamente: + - Codec mismatch G.711/G.729 entre SBC peering y core IMS. + - Tormentas de registro post-handover con CSCF tras despliegues. + - Degradación de MOS en pools SBC durante picos. + - Saturación replicada en HSS active-active. + Prioriza hipótesis respaldadas por los ejemplos del dataset incident_sft. + Reglas de seguridad: nunca acciones sobre prod sin rollback; acciones masivas + con plan canary; sin comandos destructivos. +output_schema: + type: object + required: [severity, root_cause_hypothesis, proposed_actions] + properties: + severity: {type: string, enum: [low, medium, high, critical]} + root_cause_hypothesis: {type: string} + proposed_actions: + type: array + items: + type: object + required: [id, action, target, risk_score, rollback_plan, requires_approval] +risk_threshold_for_hitl: 4 +input_label: Incident description +input_placeholder: "Sudden 80% drop in SIP registrations after deploying image 4.7.2 on the CSCF cluster." +icon: "🤖" +color: "#f59e0b" +updated_at: 2026-06-10T10:00:00Z diff --git a/common/src/forja_common/__init__.py b/common/src/forja_common/__init__.py deleted file mode 100644 index 200da0a..0000000 --- a/common/src/forja_common/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""forja_common - shared utilities (client, types, etc.).""" - -from forja_common.client import CoreClient - -__all__ = ["CoreClient"] diff --git a/common/src/forja_common/client.py b/common/src/forja_common/client.py deleted file mode 100644 index 125259d..0000000 --- a/common/src/forja_common/client.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Shared HTTP client for Forja Core API (used by web and cli).""" - -from __future__ import annotations - -import os -from typing import Any - -import httpx - - -class CoreClient: - """Synchronous HTTP client to the Forja Core API.""" - - def __init__(self, base_url: str | None = None, timeout: float = 30.0) -> None: - self.base_url = base_url or os.getenv("FORJA_CORE_URL", "http://localhost:8000") - transport = httpx.HTTPTransport(retries=2) - self._client = httpx.Client(base_url=self.base_url, timeout=timeout, transport=transport) - - def close(self) -> None: - self._client.close() - - # --- Public API methods --- - - def health(self) -> dict: - return self._get("/health") - - def list_agents(self) -> list[dict]: - return self._get("/agents") - - def get_agent(self, name: str) -> dict: - return self._get(f"/agents/{name}") - - def list_versions(self, name: str) -> list[dict]: - return self._get(f"/agents/{name}/versions") - - def diff_versions(self, name: str, v_from: str, v_to: str) -> dict: - return self._get(f"/agents/{name}/versions/{v_from}/diff/{v_to}") - - def invoke_agent(self, name: str, body: dict) -> dict: - return self._post(f"/agents/{name}/invoke", body) - - def get_execution(self, trace_id: str) -> dict: - return self._get(f"/executions/{trace_id}") - - def list_executions(self) -> list[dict]: - return self._get("/executions") - - def approve(self, trace_id: str, body: dict) -> dict: - return self._post(f"/executions/{trace_id}/approve", body) - - def reject(self, trace_id: str, body: dict) -> dict: - return self._post(f"/executions/{trace_id}/reject", body) - - def list_violations(self, **filters: Any) -> list[dict]: - params = {k: v for k, v in filters.items() if v is not None} - return self._get("/violations", params=params) - - def list_policies(self) -> list[dict]: - return self._get("/policies") - - def list_policy_versions(self, name: str) -> list[dict]: - return self._get(f"/policies/{name}/versions") - - def create_agent_version(self, name: str, agent_def: dict, message: str, author: str = "cli") -> dict: - body = {"agent": agent_def, "message": message, "author": author} - return self._post(f"/agents/{name}/versions", body) - - def create_policy_version(self, name: str, policy_def: dict, message: str, author: str = "cli") -> dict: - body = {"policy": policy_def, "message": message, "author": author} - return self._post(f"/policies/{name}/versions", body) - - def list_validators(self) -> dict: - return self._get("/policies/validators") - - # --- Internal helpers --- - - def _get(self, path: str, params: dict | None = None) -> Any: - r = self._client.get(path, params=params) - r.raise_for_status() - return r.json() - - def _post(self, path: str, body: dict) -> Any: - r = self._client.post(path, json=body) - if r.status_code in {404, 409, 422}: - return {"api_error": r.json()} - r.raise_for_status() - return r.json() diff --git a/core/requirements.txt b/core/requirements.txt index 1c470a1..ca6e69b 100644 --- a/core/requirements.txt +++ b/core/requirements.txt @@ -14,9 +14,12 @@ aiosqlite>=0.20,<0.21 presidio-analyzer>=2.2,<3.0 presidio-anonymizer>=2.2,<3.0 openai>=1.50,<2.0 +azure-ai-ml>=1.16,<2.0 +azure-identity>=1.16,<2.0 PyYAML>=6.0,<7.0 jsonschema>=4.0,<5.0 -nemoguardrails>=0.10,<0.12 +# nemoguardrails se reintroducirá cuando se integre Colang real (docs/futuro.md); +# el engine actual es un stub por keywords que no importa el paquete. # HTMX UI embebida (Opción A) jinja2>=3.1,<4.0 diff --git a/core/src/forja_core/api/agents.py b/core/src/forja_core/api/agents.py index 67e4998..926a082 100644 --- a/core/src/forja_core/api/agents.py +++ b/core/src/forja_core/api/agents.py @@ -52,10 +52,13 @@ def diff_versions(name: str, v_from: str, v_to: str, registry: RegistryDep) -> D return registry.diff_versions(name, v_from, v_to) -@router.post("/{name}/versions", response_model=AgentVersionMeta, status_code=status.HTTP_201_CREATED) -def create_agent_version(name: str, req: CreateAgentVersionRequest, registry: RegistryDep) -> AgentVersionMeta: +@router.post( + "/{name}/versions", response_model=AgentVersionMeta, status_code=status.HTTP_201_CREATED +) +def create_agent_version( + name: str, req: CreateAgentVersionRequest, registry: RegistryDep +) -> AgentVersionMeta: try: return registry.upsert_version(name, req.agent, req.message, req.author) except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) from exc - diff --git a/core/src/forja_core/api/datasets.py b/core/src/forja_core/api/datasets.py new file mode 100644 index 0000000..9930fa2 --- /dev/null +++ b/core/src/forja_core/api/datasets.py @@ -0,0 +1,39 @@ +"""Dataset registry API (read-only).""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException + +from forja_core.api.deps import DatasetStoreDep +from forja_core.domain.dataset import DatasetDefinition, DatasetVersionMeta + +router = APIRouter() + + +@router.get("", response_model=list[DatasetDefinition]) +def list_datasets(store: DatasetStoreDep) -> list[DatasetDefinition]: + return store.list_datasets() + + +@router.get("/{name}", response_model=DatasetDefinition) +def get_dataset(name: str, store: DatasetStoreDep) -> DatasetDefinition: + try: + return store.get_dataset(name) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.get("/{name}/versions", response_model=list[DatasetVersionMeta]) +def list_versions(name: str, store: DatasetStoreDep) -> list[DatasetVersionMeta]: + try: + return store.list_versions(name) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.get("/{name}/versions/{version}", response_model=DatasetDefinition) +def get_version(name: str, version: str, store: DatasetStoreDep) -> DatasetDefinition: + try: + return store.get_version(name, version) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc diff --git a/core/src/forja_core/api/deps.py b/core/src/forja_core/api/deps.py index 0fa8d03..bb5a769 100644 --- a/core/src/forja_core/api/deps.py +++ b/core/src/forja_core/api/deps.py @@ -17,10 +17,19 @@ from forja_core.guardrails.base import GuardrailEngine from forja_core.guardrails.factory import build_guardrail_engine from forja_core.llm.base import LLMProvider from forja_core.llm.factory import build_llm_provider -from forja_core.registry.factory import build_agent_registry, build_policy_store +from forja_core.registry.dataset_store import FileSystemDatasetStore +from forja_core.registry.factory import ( + build_agent_registry, + build_dataset_store, + build_model_store, + build_policy_store, +) +from forja_core.registry.model_store import FileSystemModelStore from forja_core.registry.policy_store import FileSystemPolicyStore from forja_core.registry.repository import FileSystemAgentRegistry from forja_core.runtime.orchestrator import AgentOrchestrator +from forja_core.training.base import TrainingBackend +from forja_core.training.factory import build_training_backend @lru_cache(maxsize=1) @@ -48,6 +57,21 @@ def get_guardrail_engine() -> GuardrailEngine: return build_guardrail_engine(get_settings()) +@lru_cache(maxsize=1) +def get_dataset_store() -> FileSystemDatasetStore: + return build_dataset_store(get_settings()) + + +@lru_cache(maxsize=1) +def get_model_store() -> FileSystemModelStore: + return build_model_store(get_settings()) + + +@lru_cache(maxsize=1) +def get_training_backend() -> TrainingBackend: + return build_training_backend(get_settings()) + + @lru_cache(maxsize=1) def get_orchestrator() -> AgentOrchestrator: settings = get_settings() @@ -63,3 +87,6 @@ SettingsDep = Annotated[Settings, Depends(get_settings)] RegistryDep = Annotated[FileSystemAgentRegistry, Depends(get_registry)] PolicyStoreDep = Annotated[FileSystemPolicyStore, Depends(get_policy_store)] OrchestratorDep = Annotated[AgentOrchestrator, Depends(get_orchestrator)] +DatasetStoreDep = Annotated[FileSystemDatasetStore, Depends(get_dataset_store)] +ModelStoreDep = Annotated[FileSystemModelStore, Depends(get_model_store)] +TrainingBackendDep = Annotated[TrainingBackend, Depends(get_training_backend)] diff --git a/core/src/forja_core/api/evaluation_persistence.py b/core/src/forja_core/api/evaluation_persistence.py new file mode 100644 index 0000000..b6d8f18 --- /dev/null +++ b/core/src/forja_core/api/evaluation_persistence.py @@ -0,0 +1,47 @@ +"""Persist evaluation reports under data/evaluations/.""" + +from __future__ import annotations + +import json +from pathlib import Path +from uuid import UUID + +from forja_core.domain.training import EvaluationReport + + +def _eval_dir(data_dir: Path) -> Path: + return data_dir / "evaluations" + + +def save_evaluation(data_dir: Path, report: EvaluationReport) -> None: + directory = _eval_dir(data_dir) + directory.mkdir(parents=True, exist_ok=True) + path = directory / f"{report.id}.json" + path.write_text( + json.dumps(report.model_dump(mode="json"), indent=2, default=str), + encoding="utf-8", + ) + index_path = directory / "by_training_run.json" + index: dict[str, str] = {} + if index_path.exists(): + index = json.loads(index_path.read_text(encoding="utf-8")) + index[str(report.training_run_id)] = str(report.id) + index_path.write_text(json.dumps(index, indent=2), encoding="utf-8") + + +def load_evaluation(data_dir: Path, report_id: UUID) -> EvaluationReport | None: + path = _eval_dir(data_dir) / f"{report_id}.json" + if not path.exists(): + return None + return EvaluationReport.model_validate(json.loads(path.read_text(encoding="utf-8"))) + + +def load_evaluation_for_run(data_dir: Path, training_run_id: UUID) -> EvaluationReport | None: + index_path = _eval_dir(data_dir) / "by_training_run.json" + if not index_path.exists(): + return None + index = json.loads(index_path.read_text(encoding="utf-8")) + report_id = index.get(str(training_run_id)) + if report_id is None: + return None + return load_evaluation(data_dir, UUID(report_id)) diff --git a/core/src/forja_core/api/executions.py b/core/src/forja_core/api/executions.py index a41ae0d..28f7793 100644 --- a/core/src/forja_core/api/executions.py +++ b/core/src/forja_core/api/executions.py @@ -133,12 +133,8 @@ async def invoke_agent( except FileNotFoundError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc - execution = await orchestrator.invoke( - agent_def=agent_def, policy=policy, user_input=body.input - ) - _record_execution( - settings.data_dir, str(execution.trace_id), agent_def.name, agent_def.version - ) + execution = await orchestrator.invoke(agent_def=agent_def, policy=policy, user_input=body.input) + _record_execution(settings.data_dir, str(execution.trace_id), agent_def.name, agent_def.version) if execution.status in _TERMINAL_STATUSES: append_execution(settings.data_dir, execution) for violation in execution.violations: @@ -146,23 +142,22 @@ async def invoke_agent( return execution -@router.get("", response_model=list[AgentExecutionSummary]) -async def list_executions( - registry: RegistryDep, - policies: PolicyStoreDep, - orchestrator: OrchestratorDep, - settings: SettingsDep, +async def collect_execution_summaries( + registry: FileSystemAgentRegistry, + policies: FileSystemPolicyStore, + orchestrator: AgentOrchestrator, + data_dir: Path, ) -> list[AgentExecutionSummary]: """Resumen de ejecuciones: las terminales del JSONL más las pausadas en HITL. Una ejecución en ``awaiting_approval`` no se escribe en el log append-only - (`executions.jsonl`); solo vive en el checkpointer. Para que la página de - aprobaciones del dashboard la encuentre, aquí se reconstruyen esas desde el - índice `execution_index.json` + el checkpointer. + (`executions.jsonl`); solo vive en el checkpointer. Aquí se reconstruyen esas + desde el índice `execution_index.json` + el checkpointer. Lo usan tanto la API + como la página de Approvals de la UI. """ - summaries = read_execution_summaries(settings.data_dir) + summaries = read_execution_summaries(data_dir) seen = {str(s.trace_id) for s in summaries} - for trace_id, meta in _load_index(settings.data_dir).items(): + for trace_id, meta in _load_index(data_dir).items(): if trace_id in seen: continue try: @@ -190,6 +185,16 @@ async def list_executions( return summaries +@router.get("", response_model=list[AgentExecutionSummary]) +async def list_executions( + registry: RegistryDep, + policies: PolicyStoreDep, + orchestrator: OrchestratorDep, + settings: SettingsDep, +) -> list[AgentExecutionSummary]: + return await collect_execution_summaries(registry, policies, orchestrator, settings.data_dir) + + @router.get("/{trace_id}", response_model=AgentExecution) async def get_execution( trace_id: UUID, diff --git a/core/src/forja_core/api/middlewares.py b/core/src/forja_core/api/middlewares.py index 0f32fe0..a5fc3ba 100644 --- a/core/src/forja_core/api/middlewares.py +++ b/core/src/forja_core/api/middlewares.py @@ -14,9 +14,7 @@ from forja_core.observability.logging import bind_trace_id, clear_trace_id class TraceIdMiddleware(BaseHTTPMiddleware): """Lee o genera `X-Trace-Id`, lo bind-ea al contexto de structlog y lo devuelve en la respuesta.""" - async def dispatch( - self, request: Request, call_next: RequestResponseEndpoint - ) -> Response: + async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response: trace_id = request.headers.get("X-Trace-Id") or str(uuid4()) bind_trace_id(trace_id) try: diff --git a/core/src/forja_core/api/models.py b/core/src/forja_core/api/models.py new file mode 100644 index 0000000..46fd36e --- /dev/null +++ b/core/src/forja_core/api/models.py @@ -0,0 +1,39 @@ +"""Foundation model registry API (read-only).""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException + +from forja_core.api.deps import ModelStoreDep +from forja_core.domain.foundation_model import ModelDefinition, ModelVersionMeta + +router = APIRouter() + + +@router.get("", response_model=list[ModelDefinition]) +def list_models(store: ModelStoreDep) -> list[ModelDefinition]: + return store.list_models() + + +@router.get("/{name}", response_model=ModelDefinition) +def get_model(name: str, store: ModelStoreDep) -> ModelDefinition: + try: + return store.get_model(name) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.get("/{name}/versions", response_model=list[ModelVersionMeta]) +def list_versions(name: str, store: ModelStoreDep) -> list[ModelVersionMeta]: + try: + return store.list_versions(name) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@router.get("/{name}/versions/{version}", response_model=ModelDefinition) +def get_version(name: str, version: str, store: ModelStoreDep) -> ModelDefinition: + try: + return store.get_version(name, version) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc diff --git a/core/src/forja_core/api/policies.py b/core/src/forja_core/api/policies.py index 47c7f0b..637cdd5 100644 --- a/core/src/forja_core/api/policies.py +++ b/core/src/forja_core/api/policies.py @@ -30,8 +30,12 @@ def list_versions(name: str, store: PolicyStoreDep) -> list[PolicyVersionMeta]: raise HTTPException(status_code=404, detail=str(exc)) from exc -@router.post("/{name}/versions", response_model=PolicyVersionMeta, status_code=status.HTTP_201_CREATED) -def create_policy_version(name: str, req: CreatePolicyVersionRequest, store: PolicyStoreDep) -> PolicyVersionMeta: +@router.post( + "/{name}/versions", response_model=PolicyVersionMeta, status_code=status.HTTP_201_CREATED +) +def create_policy_version( + name: str, req: CreatePolicyVersionRequest, store: PolicyStoreDep +) -> PolicyVersionMeta: try: return store.upsert_version(name, req.policy, req.message, req.author) except Exception as exc: @@ -42,6 +46,10 @@ def create_policy_version(name: str, req: CreatePolicyVersionRequest, store: Pol def list_validators() -> dict[str, list[str]]: return { "input": ["detect_pii", "prompt_injection", "toxic_language", "forbidden_topics"], - "output": ["schema_match", "pii_leakage", "forbidden_action_keywords", "telco_safety_rules"], + "output": [ + "schema_match", + "pii_leakage", + "forbidden_action_keywords", + "telco_safety_rules", + ], } - diff --git a/core/src/forja_core/api/promotion_persistence.py b/core/src/forja_core/api/promotion_persistence.py new file mode 100644 index 0000000..e3d75c1 --- /dev/null +++ b/core/src/forja_core/api/promotion_persistence.py @@ -0,0 +1,51 @@ +"""Persist promotion requests under data/promotions/.""" + +from __future__ import annotations + +import json +from pathlib import Path +from uuid import UUID + +from forja_core.domain.training import PromotionRecord, PromotionRequest, PromotionStatus + + +def _promo_dir(data_dir: Path) -> Path: + return data_dir / "promotions" + + +def save_promotion_request(data_dir: Path, request: PromotionRequest) -> None: + directory = _promo_dir(data_dir) + directory.mkdir(parents=True, exist_ok=True) + path = directory / f"{request.id}.json" + path.write_text( + json.dumps(request.model_dump(mode="json"), indent=2, default=str), + encoding="utf-8", + ) + + +def load_promotion_request(data_dir: Path, request_id: UUID) -> PromotionRequest | None: + path = _promo_dir(data_dir) / f"{request_id}.json" + if not path.exists(): + return None + return PromotionRequest.model_validate(json.loads(path.read_text(encoding="utf-8"))) + + +def list_promotion_requests( + data_dir: Path, + status: PromotionStatus | None = None, +) -> list[PromotionRequest]: + directory = _promo_dir(data_dir) + if not directory.exists(): + return [] + out: list[PromotionRequest] = [] + for path in sorted(directory.glob("*.json"), reverse=True): + req = PromotionRequest.model_validate(json.loads(path.read_text(encoding="utf-8"))) + if status is None or req.status == status: + out.append(req) + return out + + +def append_promotion_record(data_dir: Path, record: PromotionRecord) -> None: + data_dir.mkdir(parents=True, exist_ok=True) + with (data_dir / "promotion_records.jsonl").open("a", encoding="utf-8") as f: + f.write(record.model_dump_json() + "\n") diff --git a/core/src/forja_core/api/promotions.py b/core/src/forja_core/api/promotions.py new file mode 100644 index 0000000..d611a3b --- /dev/null +++ b/core/src/forja_core/api/promotions.py @@ -0,0 +1,202 @@ +"""Governance promotions: draft agent versions to active after eval gate.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Annotated +from uuid import UUID, uuid4 + +from fastapi import APIRouter, HTTPException, Query, status +from pydantic import BaseModel + +from forja_core.api.deps import RegistryDep, SettingsDep +from forja_core.api.evaluation_persistence import load_evaluation +from forja_core.api.promotion_persistence import ( + append_promotion_record, + list_promotion_requests, + load_promotion_request, + save_promotion_request, +) +from forja_core.api.training_persistence import load_training_run +from forja_core.config import Settings +from forja_core.domain.training import ( + PromotionRecord, + PromotionRequest, + PromotionStatus, + TrainingRun, +) + +router = APIRouter() + + +class CreatePromotionRequest(BaseModel): + agent_name: str + agent_version: str + training_run_id: UUID + evaluation_report_id: UUID + requested_by: str = "api" + + +class PromotionDecision(BaseModel): + approved_by: str = "api" + comment: str | None = None + + +class RejectPromotion(BaseModel): + rejected_by: str = "api" + reason: str | None = None + + +def _require_succeeded_run(run_id: UUID, settings: Settings) -> TrainingRun: + run = load_training_run(settings.data_dir, run_id) + if run is None: + raise HTTPException(status_code=404, detail="training run not found") + if run.status != "succeeded": + raise HTTPException( + status_code=409, + detail=f"training run status '{run.status}' must be 'succeeded' for this operation", + ) + return run + + +@router.get("", response_model=list[PromotionRequest]) +def list_promotions( + settings: SettingsDep, + status: Annotated[PromotionStatus | None, Query()] = None, +) -> list[PromotionRequest]: + return list_promotion_requests(settings.data_dir, status=status) + + +@router.get("/{request_id}", response_model=PromotionRequest) +def get_promotion(request_id: UUID, settings: SettingsDep) -> PromotionRequest: + req = load_promotion_request(settings.data_dir, request_id) + if req is None: + raise HTTPException(status_code=404, detail="promotion request not found") + return req + + +@router.post("", response_model=PromotionRequest, status_code=status.HTTP_201_CREATED) +def request_promotion( + body: CreatePromotionRequest, + settings: SettingsDep, + registry: RegistryDep, +) -> PromotionRequest: + _require_succeeded_run(body.training_run_id, settings) + + report = load_evaluation(settings.data_dir, body.evaluation_report_id) + if report is None: + raise HTTPException(status_code=404, detail="evaluation report not found") + if report.training_run_id != body.training_run_id: + raise HTTPException(status_code=422, detail="evaluation report does not match training run") + if not report.passed: + raise HTTPException( + status_code=422, + detail="evaluation did not pass; promotion blocked by guardrail gate", + ) + + try: + agent = registry.get_version(body.agent_name, body.agent_version) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + if agent.state == "active": + raise HTTPException(status_code=409, detail="agent version is already active") + if agent.name != report.agent_name: + raise HTTPException( + status_code=422, detail="evaluation agent does not match promotion target" + ) + if report.agent_version != body.agent_version: + raise HTTPException( + status_code=422, + detail=( + f"evaluation covered version '{report.agent_version}' but promotion " + f"targets '{body.agent_version}'; evaluate the candidate version first" + ), + ) + + now = datetime.now(UTC) + req = PromotionRequest( + id=uuid4(), + agent_name=body.agent_name, + agent_version=body.agent_version, + training_run_id=body.training_run_id, + evaluation_report_id=body.evaluation_report_id, + status="pending_approval", + requested_by=body.requested_by, + created_at=now, + updated_at=now, + ) + save_promotion_request(settings.data_dir, req) + return req + + +@router.post("/{request_id}/approve", response_model=PromotionRecord) +def approve_promotion( + request_id: UUID, + body: PromotionDecision, + settings: SettingsDep, + registry: RegistryDep, +) -> PromotionRecord: + req = load_promotion_request(settings.data_dir, request_id) + if req is None: + raise HTTPException(status_code=404, detail="promotion request not found") + if req.status != "pending_approval": + raise HTTPException(status_code=409, detail=f"promotion status is '{req.status}'") + + report = load_evaluation(settings.data_dir, req.evaluation_report_id) + if report is None or not report.passed: + raise HTTPException(status_code=422, detail="evaluation gate no longer satisfied") + + registry.promote_to_active( + req.agent_name, + req.agent_version, + author=body.approved_by, + comment=body.comment or "Approved via governance promotion", + ) + + now = datetime.now(UTC) + updated_req = req.model_copy( + update={ + "status": "approved", + "decided_by": body.approved_by, + "comment": body.comment, + "updated_at": now, + }, + ) + save_promotion_request(settings.data_dir, updated_req) + + record = PromotionRecord( + id=uuid4(), + agent_name=req.agent_name, + agent_version=req.agent_version, + training_run_id=req.training_run_id, + evaluation_report_id=req.evaluation_report_id, + approved_by=body.approved_by, + comment=body.comment, + created_at=now, + ) + append_promotion_record(settings.data_dir, record) + return record + + +@router.post("/{request_id}/reject", response_model=PromotionRequest) +def reject_promotion( + request_id: UUID, + body: RejectPromotion, + settings: SettingsDep, +) -> PromotionRequest: + req = load_promotion_request(settings.data_dir, request_id) + if req is None: + raise HTTPException(status_code=404, detail="promotion request not found") + if req.status != "pending_approval": + raise HTTPException(status_code=409, detail=f"promotion status is '{req.status}'") + + updated = req.model_copy( + update={ + "status": "rejected", + "decided_by": body.rejected_by, + "comment": body.reason, + "updated_at": datetime.now(UTC), + }, + ) + save_promotion_request(settings.data_dir, updated) + return updated diff --git a/core/src/forja_core/api/training.py b/core/src/forja_core/api/training.py new file mode 100644 index 0000000..bd3cc97 --- /dev/null +++ b/core/src/forja_core/api/training.py @@ -0,0 +1,237 @@ +"""Training runs API — submit jobs to external MLOps backends.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +from fastapi import APIRouter, HTTPException, status +from pydantic import BaseModel, field_validator + +from forja_core.api.deps import ( + DatasetStoreDep, + ModelStoreDep, + OrchestratorDep, + PolicyStoreDep, + RegistryDep, + SettingsDep, + TrainingBackendDep, +) +from forja_core.api.evaluation_persistence import load_evaluation_for_run, save_evaluation +from forja_core.api.training_persistence import ( + list_training_runs, + load_training_run, + save_training_run, +) +from forja_core.domain.training import ( + EvaluationReport, + TrainingHyperparameters, + TrainingJobSpec, + TrainingRun, +) +from forja_core.evaluation.post_train import run_post_train_evaluation +from forja_core.registry.dataset_store import FileSystemDatasetStore +from forja_core.registry.model_store import FileSystemModelStore +from forja_core.registry.repository import FileSystemAgentRegistry +from forja_core.training.base import TrainingBackend + +router = APIRouter() + + +class CreateTrainingRunRequest(BaseModel): + dataset_name: str + dataset_version: str | None = None + model_name: str + model_version: str | None = None + agent_name: str | None = None + agent_version: str | None = None + hyperparameters: TrainingHyperparameters | None = None + + @field_validator( + "dataset_version", "model_version", "agent_name", "agent_version", mode="before" + ) + @classmethod + def _empty_string_is_none(cls, value: object) -> object: + # HTML form selects submit "" for the unselected option. + return value or None + + +def _resolve_spec( + body: CreateTrainingRunRequest, + datasets: FileSystemDatasetStore, + models: FileSystemModelStore, + registry: FileSystemAgentRegistry, +) -> TrainingJobSpec: + try: + dataset = datasets.get_dataset(body.dataset_name, body.dataset_version) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + try: + model = models.get_model(body.model_name, body.model_version) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + if body.agent_version and not body.agent_name: + raise HTTPException(status_code=422, detail="agent_version requires agent_name") + if body.agent_name: + try: + registry.get_agent(body.agent_name, body.agent_version) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + return TrainingJobSpec( + dataset_name=dataset.name, + dataset_version=dataset.version, + model_name=model.name, + model_version=model.version, + agent_name=body.agent_name, + agent_version=body.agent_version, + hyperparameters=body.hyperparameters or TrainingHyperparameters(), + ) + + +async def _apply_backend_status( + backend: TrainingBackend, + run: TrainingRun, +) -> TrainingRun: + if run.external_job_id is None: + return run + job = await backend.status(run.external_job_id) + now = datetime.now(UTC) + return run.model_copy( + update={ + "status": job.status, + "updated_at": now, + "artifact_refs": job.artifacts, + "error_message": job.message if job.status == "failed" else None, + }, + ) + + +@router.get("", response_model=list[TrainingRun]) +def list_runs(settings: SettingsDep) -> list[TrainingRun]: + return list_training_runs(settings.data_dir) + + +@router.get("/{run_id}", response_model=TrainingRun) +def get_run(run_id: UUID, settings: SettingsDep) -> TrainingRun: + run = load_training_run(settings.data_dir, run_id) + if run is None: + raise HTTPException(status_code=404, detail="training run not found") + return run + + +@router.post("", response_model=TrainingRun, status_code=status.HTTP_201_CREATED) +async def create_run( + body: CreateTrainingRunRequest, + settings: SettingsDep, + datasets: DatasetStoreDep, + models: ModelStoreDep, + registry: RegistryDep, + backend: TrainingBackendDep, +) -> TrainingRun: + spec = _resolve_spec(body, datasets, models, registry) + now = datetime.now(UTC) + run_id = uuid4() + run = TrainingRun( + id=run_id, + backend=settings.training_backend, + status="submitted", + spec=spec, + created_at=now, + updated_at=now, + ) + external_id = await backend.submit(spec, run_id) + run = run.model_copy( + update={ + "external_job_id": external_id, + "status": "queued", + "updated_at": datetime.now(UTC), + }, + ) + run = await _apply_backend_status(backend, run) + save_training_run(settings.data_dir, run) + return run + + +@router.post("/{run_id}/refresh", response_model=TrainingRun) +async def refresh_run( + run_id: UUID, + settings: SettingsDep, + backend: TrainingBackendDep, +) -> TrainingRun: + run = load_training_run(settings.data_dir, run_id) + if run is None: + raise HTTPException(status_code=404, detail="training run not found") + if run.backend != settings.training_backend: + raise HTTPException( + status_code=409, + detail=( + f"run backend '{run.backend}' does not match " + f"configured '{settings.training_backend}'" + ), + ) + run = await _apply_backend_status(backend, run) + save_training_run(settings.data_dir, run) + return run + + +def _require_succeeded_run(run_id: UUID, settings: SettingsDep) -> TrainingRun: + run = load_training_run(settings.data_dir, run_id) + if run is None: + raise HTTPException(status_code=404, detail="training run not found") + if run.status != "succeeded": + raise HTTPException( + status_code=409, + detail=f"training run status '{run.status}' must be 'succeeded'", + ) + return run + + +@router.post("/{run_id}/evaluate", response_model=EvaluationReport) +async def evaluate_run( + run_id: UUID, + settings: SettingsDep, + registry: RegistryDep, + policies: PolicyStoreDep, + orchestrator: OrchestratorDep, +) -> EvaluationReport: + run = _require_succeeded_run(run_id, settings) + agent_name = run.spec.agent_name + if not agent_name: + raise HTTPException( + status_code=422, + detail="training run has no agent_name; set it when creating the run", + ) + + existing = load_evaluation_for_run(settings.data_dir, run_id) + if existing is not None: + return existing + + # Evaluate the candidate version tied to the run (falls back to active). + try: + agent_def = registry.get_agent(agent_name, run.spec.agent_version) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + if not agent_def.guardrails: + raise HTTPException(status_code=422, detail="agent has no guardrail policy") + try: + policy = policies.get_policy(agent_def.guardrails[0]) + except FileNotFoundError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + + report = await run_post_train_evaluation( + training_run_id=run_id, + agent_def=agent_def, + policy=policy, + orchestrator=orchestrator, + agents_dir=settings.agents_dir, + ) + save_evaluation(settings.data_dir, report) + return report + + +@router.get("/{run_id}/evaluation", response_model=EvaluationReport) +def get_run_evaluation(run_id: UUID, settings: SettingsDep) -> EvaluationReport: + report = load_evaluation_for_run(settings.data_dir, run_id) + if report is None: + raise HTTPException(status_code=404, detail="evaluation not found for this training run") + return report diff --git a/core/src/forja_core/api/training_persistence.py b/core/src/forja_core/api/training_persistence.py new file mode 100644 index 0000000..e6c9968 --- /dev/null +++ b/core/src/forja_core/api/training_persistence.py @@ -0,0 +1,42 @@ +"""Persist training runs as one JSON file per run under data/training_runs/.""" + +from __future__ import annotations + +import json +from pathlib import Path +from uuid import UUID + +from forja_core.domain.training import TrainingRun + + +def _runs_dir(data_dir: Path) -> Path: + return data_dir / "training_runs" + + +def save_training_run(data_dir: Path, run: TrainingRun) -> None: + directory = _runs_dir(data_dir) + directory.mkdir(parents=True, exist_ok=True) + path = directory / f"{run.id}.json" + path.write_text( + json.dumps(run.model_dump(mode="json"), indent=2, default=str), + encoding="utf-8", + ) + + +def load_training_run(data_dir: Path, run_id: UUID) -> TrainingRun | None: + path = _runs_dir(data_dir) / f"{run_id}.json" + if not path.exists(): + return None + data = json.loads(path.read_text(encoding="utf-8")) + return TrainingRun.model_validate(data) + + +def list_training_runs(data_dir: Path) -> list[TrainingRun]: + directory = _runs_dir(data_dir) + if not directory.exists(): + return [] + runs: list[TrainingRun] = [] + for path in sorted(directory.glob("*.json"), reverse=True): + data = json.loads(path.read_text(encoding="utf-8")) + runs.append(TrainingRun.model_validate(data)) + return runs diff --git a/core/src/forja_core/config.py b/core/src/forja_core/config.py index 996bf5b..bcfdde4 100644 --- a/core/src/forja_core/config.py +++ b/core/src/forja_core/config.py @@ -35,16 +35,26 @@ class Settings(BaseSettings): # Guardrails guardrails_nemo_enabled: bool = False + # Comma-separated keywords for the NeMo topical-rails stub. Empty disables + # the off-topic check (no domain assumptions baked into the core). + guardrails_nemo_allowed_keywords: str = "" # Observabilidad log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR"] = "INFO" + # Training / MLOps orchestration + training_backend: Literal["mock", "azure_ml"] = "mock" + azure_ml_subscription_id: str = "" + azure_ml_resource_group: str = "" + azure_ml_workspace_name: str = "" + azure_ml_compute: str = "" + azure_ml_environment: str = "AzureML-sklearn-1.5" + azure_ml_experiment_name: str = "forja-training" + azure_ml_command: str = "python train.py" + # Persistencia data_dir: Path = Field(default=Path("./data")) agents_dir: Path = Field(default=Path("./agents")) policies_dir: Path = Field(default=Path("./policies")) - - -def get_settings() -> Settings: - """Helper para inyección por dependencia en FastAPI.""" - return Settings() + datasets_dir: Path = Field(default=Path("./datasets")) + models_dir: Path = Field(default=Path("./models")) diff --git a/core/src/forja_core/domain/agent.py b/core/src/forja_core/domain/agent.py index cf54e20..e47b15d 100644 --- a/core/src/forja_core/domain/agent.py +++ b/core/src/forja_core/domain/agent.py @@ -7,6 +7,8 @@ from typing import Any, Literal from pydantic import BaseModel, Field +from forja_core.domain.version_meta import VersionMeta + class LLMConfig(BaseModel): """Configuración del proveedor LLM utilizado por un agente.""" @@ -17,14 +19,8 @@ class LLMConfig(BaseModel): max_tokens: int = Field(default=2000, ge=1, le=128_000) -class AgentVersionMeta(BaseModel): - """Metadatos de una versión concreta de un agente (estilo commit Git).""" - - id: str - hash: str - author: str - message: str - created_at: datetime +# Alias de compatibilidad: los metadatos de versión son comunes a todos los registries. +AgentVersionMeta = VersionMeta class AgentDefinition(BaseModel): diff --git a/core/src/forja_core/domain/dataset.py b/core/src/forja_core/domain/dataset.py new file mode 100644 index 0000000..8143e3a --- /dev/null +++ b/core/src/forja_core/domain/dataset.py @@ -0,0 +1,28 @@ +"""Dataset definitions for fine-tuning (versioned YAML).""" + +from __future__ import annotations + +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, Field + +from forja_core.domain.version_meta import VersionMeta + +# Compatibility alias: version metadata is shared across all registries. +DatasetVersionMeta = VersionMeta + + +class DatasetDefinition(BaseModel): + """Declarative dataset used to submit training jobs.""" + + name: str + version: str + owner: str + purpose: str + state: Literal["draft", "active", "deprecated"] = "active" + format: Literal["jsonl", "conversation"] = "jsonl" + records_path: str = Field( + description="Path relative to the dataset directory, e.g. records/train.jsonl", + ) + updated_at: datetime diff --git a/core/src/forja_core/domain/foundation_model.py b/core/src/forja_core/domain/foundation_model.py new file mode 100644 index 0000000..dfe880d --- /dev/null +++ b/core/src/forja_core/domain/foundation_model.py @@ -0,0 +1,28 @@ +"""Base / fine-tune target model definitions (versioned YAML).""" + +from __future__ import annotations + +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, Field + +from forja_core.domain.version_meta import VersionMeta + +# Compatibility alias: version metadata is shared across all registries. +ModelVersionMeta = VersionMeta + + +class ModelDefinition(BaseModel): + """Registered base model or training target reference.""" + + name: str + version: str + owner: str + purpose: str + state: Literal["draft", "active", "deprecated"] = "active" + base_model_id: str = Field( + description="Provider model id, e.g. HuggingFace repo or Azure deployment name", + ) + adaptation: Literal["full", "lora", "qlora"] = "lora" + updated_at: datetime diff --git a/core/src/forja_core/domain/policy.py b/core/src/forja_core/domain/policy.py index 439d0f3..069426d 100644 --- a/core/src/forja_core/domain/policy.py +++ b/core/src/forja_core/domain/policy.py @@ -2,11 +2,12 @@ from __future__ import annotations -from datetime import datetime from typing import Any, Literal from pydantic import BaseModel, Field +from forja_core.domain.version_meta import VersionMeta + class PolicyValidator(BaseModel): """Validador individual configurado en una política.""" @@ -15,14 +16,8 @@ class PolicyValidator(BaseModel): config: dict[str, Any] = Field(default_factory=dict) -class PolicyVersionMeta(BaseModel): - """Metadatos de una versión concreta de una política (estilo commit Git).""" - - id: str - hash: str - author: str - message: str - created_at: datetime +# Alias de compatibilidad: los metadatos de versión son comunes a todos los registries. +PolicyVersionMeta = VersionMeta class PolicyDefinition(BaseModel): diff --git a/core/src/forja_core/domain/training.py b/core/src/forja_core/domain/training.py new file mode 100644 index 0000000..c924cd0 --- /dev/null +++ b/core/src/forja_core/domain/training.py @@ -0,0 +1,127 @@ +"""Training jobs, runs, and promotion-related records.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, Field + +TrainingRunStatus = Literal[ + "submitted", + "queued", + "running", + "succeeded", + "failed", + "cancelled", +] +TrainingBackendKind = Literal["mock", "azure_ml"] + + +class TrainingHyperparameters(BaseModel): + """Minimal hyperparameters passed to an external trainer.""" + + learning_rate: float = Field(default=1e-4, gt=0.0, le=1.0) + epochs: int = Field(default=1, ge=1, le=100) + batch_size: int = Field(default=4, ge=1, le=512) + + +class TrainingJobSpec(BaseModel): + """Input specification for a training backend.""" + + dataset_name: str + dataset_version: str + model_name: str + model_version: str + agent_name: str | None = None + # Candidate agent version this run is meant to produce/validate. Post-train + # evaluation runs against this version, and promotion requires the + # evaluation report to match it. + agent_version: str | None = None + hyperparameters: TrainingHyperparameters = Field(default_factory=TrainingHyperparameters) + + +class ArtifactRef(BaseModel): + """Output artifact from a completed training job.""" + + uri: str + kind: Literal["checkpoint", "adapter", "metrics"] = "adapter" + + +class BackendJobStatus(BaseModel): + """Status returned by an external MLOps backend.""" + + status: TrainingRunStatus + message: str | None = None + artifacts: list[ArtifactRef] = Field(default_factory=list) + + +class TrainingRun(BaseModel): + """Persisted training run orchestrated by Forja.""" + + id: UUID + backend: TrainingBackendKind + status: TrainingRunStatus + spec: TrainingJobSpec + external_job_id: str | None = None + created_at: datetime + updated_at: datetime + artifact_refs: list[ArtifactRef] = Field(default_factory=list) + error_message: str | None = None + + +class ScenarioEvalResult(BaseModel): + """Result of running one canonical scenario through the governed runtime.""" + + scenario: str + status: str + blocking_violations: int = 0 + + +class EvaluationReport(BaseModel): + """Post-training evaluation summary (gate before promotion).""" + + id: UUID + training_run_id: UUID + agent_name: str + agent_version: str + passed: bool + scenario_count: int = 0 + violation_count: int = 0 + scenarios: list[ScenarioEvalResult] = Field(default_factory=list) + created_at: datetime + notes: str | None = None + + +PromotionStatus = Literal["pending_approval", "approved", "rejected"] + + +class PromotionRequest(BaseModel): + """Governance HITL request to promote a draft agent version to active.""" + + id: UUID + agent_name: str + agent_version: str + training_run_id: UUID + evaluation_report_id: UUID + status: PromotionStatus + requested_by: str + created_at: datetime + updated_at: datetime + # Operator who approved or rejected (audit trail for both outcomes). + decided_by: str | None = None + comment: str | None = None + + +class PromotionRecord(BaseModel): + """Audit record when a model/agent version is promoted to active.""" + + id: UUID + agent_name: str + agent_version: str + training_run_id: UUID | None = None + evaluation_report_id: UUID | None = None + approved_by: str + comment: str | None = None + created_at: datetime diff --git a/core/src/forja_core/domain/version_meta.py b/core/src/forja_core/domain/version_meta.py new file mode 100644 index 0000000..b973c20 --- /dev/null +++ b/core/src/forja_core/domain/version_meta.py @@ -0,0 +1,17 @@ +"""Metadatos de versión compartidos por todos los registries versionados.""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel + + +class VersionMeta(BaseModel): + """Metadatos de una versión concreta de un artefacto versionado (estilo commit Git).""" + + id: str + hash: str + author: str + message: str + created_at: datetime diff --git a/core/src/forja_core/evaluation/__init__.py b/core/src/forja_core/evaluation/__init__.py new file mode 100644 index 0000000..f825cd1 --- /dev/null +++ b/core/src/forja_core/evaluation/__init__.py @@ -0,0 +1 @@ +"""Post-training evaluation and promotion gates.""" diff --git a/core/src/forja_core/evaluation/post_train.py b/core/src/forja_core/evaluation/post_train.py new file mode 100644 index 0000000..1a0a26f --- /dev/null +++ b/core/src/forja_core/evaluation/post_train.py @@ -0,0 +1,85 @@ +"""Run post-training evaluation: governed agent runs over canonical scenarios.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from uuid import UUID, uuid4 + +from forja_core.domain.agent import AgentDefinition +from forja_core.domain.policy import PolicyDefinition +from forja_core.domain.training import EvaluationReport, ScenarioEvalResult +from forja_core.evaluation.scenarios import load_agent_scenarios +from forja_core.runtime.orchestrator import AgentOrchestrator + + +def _blocking_violation_count(execution: object) -> int: + violations = getattr(execution, "violations", []) + return sum(1 for v in violations if v.blocked) + + +async def run_post_train_evaluation( + *, + training_run_id: UUID, + agent_def: AgentDefinition, + policy: PolicyDefinition, + orchestrator: AgentOrchestrator, + agents_dir: Path, +) -> EvaluationReport: + """Execute each scenario; pass when there are no blocking guardrail violations.""" + scenarios = load_agent_scenarios(agents_dir, agent_def.name) + if not scenarios: + now = datetime.now(UTC) + return EvaluationReport( + id=uuid4(), + training_run_id=training_run_id, + agent_name=agent_def.name, + agent_version=agent_def.version, + passed=False, + scenario_count=0, + violation_count=0, + created_at=now, + notes="No scenarios found under agents//examples/", + ) + + results: list[ScenarioEvalResult] = [] + total_blocking = 0 + for name, user_input in scenarios: + execution = await orchestrator.invoke( + agent_def=agent_def, + policy=policy, + user_input=user_input, + ) + blocking = _blocking_violation_count(execution) + total_blocking += blocking + results.append( + ScenarioEvalResult( + scenario=name, + status=execution.status, + blocking_violations=blocking, + ) + ) + + failed_count = sum(1 for r in results if r.status == "failed") + passed = total_blocking == 0 and failed_count == 0 + notes = None + if not passed: + parts: list[str] = [] + if total_blocking: + parts.append(f"{total_blocking} blocking guardrail violation(s)") + if failed_count: + parts.append(f"{failed_count} scenario(s) failed") + notes = "; ".join(parts) + + return EvaluationReport( + id=uuid4(), + training_run_id=training_run_id, + agent_name=agent_def.name, + agent_version=agent_def.version, + passed=passed, + scenario_count=len(results), + violation_count=total_blocking, + scenarios=results, + created_at=datetime.now(UTC), + notes=notes, + ) diff --git a/core/src/forja_core/evaluation/scenarios.py b/core/src/forja_core/evaluation/scenarios.py new file mode 100644 index 0000000..664afda --- /dev/null +++ b/core/src/forja_core/evaluation/scenarios.py @@ -0,0 +1,16 @@ +"""Load canonical evaluation scenarios from agent example files.""" + +from __future__ import annotations + +from pathlib import Path + + +def load_agent_scenarios(agents_dir: Path, agent_name: str) -> list[tuple[str, str]]: + """Return (scenario_name, input_text) pairs from agents//examples/*.txt.""" + examples_dir = agents_dir / agent_name / "examples" + if not examples_dir.is_dir(): + return [] + scenarios: list[tuple[str, str]] = [] + for path in sorted(examples_dir.glob("*.txt")): + scenarios.append((path.name, path.read_text(encoding="utf-8"))) + return scenarios diff --git a/core/src/forja_core/guardrails/factory.py b/core/src/forja_core/guardrails/factory.py index 78f1b2c..50591ca 100644 --- a/core/src/forja_core/guardrails/factory.py +++ b/core/src/forja_core/guardrails/factory.py @@ -10,24 +10,15 @@ from forja_core.guardrails.nemo import NeMoGuardrailsEngine def build_guardrail_engine(settings: Settings) -> GuardrailEngine: - """Ensambla el engine: GuardrailsAIEngine siempre; NeMo si está habilitado.""" + """Ensambla el engine: GuardrailsAIEngine siempre; NeMo si está habilitado. + + Los keywords de topical rails vienen de configuración (CSV en + ``GUARDRAILS_NEMO_ALLOWED_KEYWORDS``); el core no asume ningún dominio. + """ engines: list[GuardrailEngine] = [GuardrailsAIEngine()] if settings.guardrails_nemo_enabled: - engines.append( - NeMoGuardrailsEngine( - allowed_keywords=[ - "sip", - "ims", - "cscf", - "sbc", - "mos", - "hss", - "registro", - "incidente", - "voz", - "codec", - "rollback", - ], - ) - ) + keywords = [ + k.strip() for k in settings.guardrails_nemo_allowed_keywords.split(",") if k.strip() + ] + engines.append(NeMoGuardrailsEngine(allowed_keywords=keywords)) return CompositeGuardrailEngine(engines) diff --git a/core/src/forja_core/guardrails/nemo.py b/core/src/forja_core/guardrails/nemo.py index 23a8ba9..f60359b 100644 --- a/core/src/forja_core/guardrails/nemo.py +++ b/core/src/forja_core/guardrails/nemo.py @@ -48,7 +48,7 @@ class NeMoGuardrailsEngine: stage="input", validator="NeMoTopicalRails", severity="warning", - message="Input fuera de los temas permitidos.", + message="Input outside the allowed topics.", blocked=False, ) ] diff --git a/core/src/forja_core/guardrails/validators.py b/core/src/forja_core/guardrails/validators.py index 3f17563..bca7ae1 100644 --- a/core/src/forja_core/guardrails/validators.py +++ b/core/src/forja_core/guardrails/validators.py @@ -96,7 +96,7 @@ def detect_pii( stage=stage, validator="DetectPII", severity=sev, - message=f"PII detectada: {matched}", + message=f"PII detected: {matched}", blocked=blocked, ) ] @@ -123,7 +123,7 @@ def _pii_regex_fallback( stage=stage, validator="DetectPII", severity=sev, - message=f"PII detectada (regex fallback): {', '.join(matched)}", + message=f"PII detected (regex fallback): {', '.join(matched)}", blocked=blocked, ) ] @@ -144,7 +144,7 @@ def prompt_injection( stage=stage, validator="PromptInjection", severity=sev, - message=f"Patrón de inyección detectado: {pattern}", + message=f"Injection pattern detected: {pattern}", blocked=blocked, ) ] @@ -175,7 +175,7 @@ def toxic_language( stage=stage, validator="ToxicLanguage", severity=sev, - message=f"Lenguaje tóxico (score={score:.2f}, hits={hits})", + message=f"Toxic language (score={score:.2f}, hits={hits})", blocked=blocked, ) ] @@ -198,7 +198,7 @@ def forbidden_topics( stage=stage, validator="ForbiddenTopics", severity=sev, - message=f"Temas prohibidos: {', '.join(matched)}", + message=f"Forbidden topics: {', '.join(matched)}", blocked=blocked, ) ] @@ -256,7 +256,7 @@ def forbidden_action_keywords( stage=stage, validator="ForbiddenActionKeywords", severity=sev, - message=f"Palabras prohibidas en acciones: {', '.join(matched)}", + message=f"Forbidden keywords in actions: {', '.join(matched)}", blocked=blocked, ) ] @@ -275,7 +275,7 @@ def telco_safety_rules( target = str(a.get("target", "")).lower() rollback = str(a.get("rollback_plan", "")).strip() if ("prod" in target or "production" in target) and (not rollback or rollback == "n/a"): - issues.append(f"acción sobre prod sin rollback: {a.get('action')}") + issues.append(f"production action without rollback: {a.get('action')}") if "never_propose_mass_action_without_canary" in rules: mass_keywords = {"all", "todos", "*", "mass"} @@ -283,7 +283,7 @@ def telco_safety_rules( target = str(a.get("target", "")).lower() plan = str(a.get("rollback_plan", "")).lower() if any(k in target for k in mass_keywords) and "canary" not in plan: - issues.append(f"acción masiva sin canary: {a.get('action')}") + issues.append(f"mass action without canary plan: {a.get('action')}") if not issues: return [] diff --git a/core/src/forja_core/llm/factory.py b/core/src/forja_core/llm/factory.py index 379bbc9..f33d00c 100644 --- a/core/src/forja_core/llm/factory.py +++ b/core/src/forja_core/llm/factory.py @@ -2,16 +2,18 @@ from __future__ import annotations +from typing import Literal + from forja_core.config import Settings from forja_core.llm.azure import AzureOpenAIProvider from forja_core.llm.base import LLMProvider +from forja_core.llm.fallback import FallbackLLMProvider from forja_core.llm.mock import MockProvider from forja_core.llm.openai import OpenAIProvider -def build_llm_provider(settings: Settings) -> LLMProvider: - """Materializa el proveedor activo. Falla en arranque si la config es inconsistente.""" - match settings.llm_provider: +def _build_single(provider: Literal["mock", "azure", "openai"], settings: Settings) -> LLMProvider: + match provider: case "mock": return MockProvider() case "azure": @@ -26,3 +28,13 @@ def build_llm_provider(settings: Settings) -> LLMProvider: api_key=settings.openai_api_key, model=settings.openai_model, ) + + +def build_llm_provider(settings: Settings) -> LLMProvider: + """Materializa el proveedor activo (con fallback opcional). Falla en arranque si la config es inconsistente.""" + primary = _build_single(settings.llm_provider, settings) + fallback_name = settings.llm_fallback_provider + if not fallback_name or fallback_name == settings.llm_provider: + return primary + fallback = _build_single(fallback_name, settings) + return FallbackLLMProvider(primary, fallback) diff --git a/core/src/forja_core/llm/fallback.py b/core/src/forja_core/llm/fallback.py new file mode 100644 index 0000000..f7316d0 --- /dev/null +++ b/core/src/forja_core/llm/fallback.py @@ -0,0 +1,46 @@ +"""Proveedor LLM con fallback: intenta el primario y, si agota sus reintentos, usa el secundario.""" + +from __future__ import annotations + +from typing import Any + +import structlog + +from forja_core.llm.base import CompletionResult, LLMProvider, Message + +log = structlog.get_logger(__name__) + + +class FallbackLLMProvider: + """Envuelve dos providers: si el primario lanza, delega en el de fallback. + + Cada provider interno conserva su propia política de retries; este wrapper + solo decide el cambio de proveedor cuando el primario falla definitivamente. + """ + + def __init__(self, primary: LLMProvider, fallback: LLMProvider) -> None: + self._primary = primary + self._fallback = fallback + self.name = f"{primary.name}+fallback:{fallback.name}" + + async def complete( + self, + messages: list[Message], + schema: dict[str, Any] | None = None, + temperature: float = 0.2, + max_tokens: int = 2000, + ) -> CompletionResult: + try: + return await self._primary.complete( + messages, schema=schema, temperature=temperature, max_tokens=max_tokens + ) + except Exception as exc: + log.warning( + "llm_primary_failed_falling_back", + primary=self._primary.name, + fallback=self._fallback.name, + error=str(exc), + ) + return await self._fallback.complete( + messages, schema=schema, temperature=temperature, max_tokens=max_tokens + ) diff --git a/core/src/forja_core/main.py b/core/src/forja_core/main.py index 320d4d7..c8da2cd 100644 --- a/core/src/forja_core/main.py +++ b/core/src/forja_core/main.py @@ -24,7 +24,16 @@ def create_app() -> FastAPI: async def health() -> dict[str, str]: return {"status": "ok"} - from forja_core.api import agents, executions, policies, violations + from forja_core.api import ( + agents, + datasets, + executions, + models, + policies, + promotions, + training, + violations, + ) from forja_core.web import ui # UI HTMX embebida (Opción A) en rutas amigables para humanos. @@ -36,6 +45,10 @@ def create_app() -> FastAPI: app.include_router(executions.router, prefix="/api/executions", tags=["executions"]) app.include_router(policies.router, prefix="/api/policies", tags=["policies"]) app.include_router(violations.router, prefix="/api/violations", tags=["violations"]) + app.include_router(datasets.router, prefix="/api/datasets", tags=["datasets"]) + app.include_router(models.router, prefix="/api/models", tags=["models"]) + app.include_router(training.router, prefix="/api/training/runs", tags=["training"]) + app.include_router(promotions.router, prefix="/api/promotions", tags=["promotions"]) return app diff --git a/core/src/forja_core/registry/dataset_store.py b/core/src/forja_core/registry/dataset_store.py new file mode 100644 index 0000000..585d08e --- /dev/null +++ b/core/src/forja_core/registry/dataset_store.py @@ -0,0 +1,25 @@ +"""Read-only dataset registry (YAML versioned, same layout as agents/).""" + +from __future__ import annotations + +from forja_core.domain.dataset import DatasetDefinition, DatasetVersionMeta +from forja_core.registry.yaml_store import VersionedYamlStore + + +class FileSystemDatasetStore(VersionedYamlStore[DatasetDefinition]): + """Reads datasets from datasets//{index.yaml,versions/vN.yaml}.""" + + model_cls = DatasetDefinition + kind = "dataset" + + def list_datasets(self) -> list[DatasetDefinition]: + return self._list_all() + + def get_dataset(self, name: str, version: str | None = None) -> DatasetDefinition: + return self._get(name, version) + + def get_version(self, name: str, version: str) -> DatasetDefinition: + return self._get_version(name, version) + + def list_versions(self, name: str) -> list[DatasetVersionMeta]: + return self._list_versions(name) diff --git a/core/src/forja_core/registry/factory.py b/core/src/forja_core/registry/factory.py index 54f92b6..47ff940 100644 --- a/core/src/forja_core/registry/factory.py +++ b/core/src/forja_core/registry/factory.py @@ -3,6 +3,8 @@ from __future__ import annotations from forja_core.config import Settings +from forja_core.registry.dataset_store import FileSystemDatasetStore +from forja_core.registry.model_store import FileSystemModelStore from forja_core.registry.policy_store import FileSystemPolicyStore from forja_core.registry.repository import FileSystemAgentRegistry @@ -15,3 +17,11 @@ def build_agent_registry(settings: Settings) -> FileSystemAgentRegistry: def build_policy_store(settings: Settings) -> FileSystemPolicyStore: """Construye el store de políticas apuntando a ``settings.policies_dir``.""" return FileSystemPolicyStore(settings.policies_dir) + + +def build_dataset_store(settings: Settings) -> FileSystemDatasetStore: + return FileSystemDatasetStore(settings.datasets_dir) + + +def build_model_store(settings: Settings) -> FileSystemModelStore: + return FileSystemModelStore(settings.models_dir) diff --git a/core/src/forja_core/registry/model_store.py b/core/src/forja_core/registry/model_store.py new file mode 100644 index 0000000..ae24e55 --- /dev/null +++ b/core/src/forja_core/registry/model_store.py @@ -0,0 +1,25 @@ +"""Read-only foundation model registry (YAML versioned).""" + +from __future__ import annotations + +from forja_core.domain.foundation_model import ModelDefinition, ModelVersionMeta +from forja_core.registry.yaml_store import VersionedYamlStore + + +class FileSystemModelStore(VersionedYamlStore[ModelDefinition]): + """Reads models from models//{index.yaml,versions/vN.yaml}.""" + + model_cls = ModelDefinition + kind = "model" + + def list_models(self) -> list[ModelDefinition]: + return self._list_all() + + def get_model(self, name: str, version: str | None = None) -> ModelDefinition: + return self._get(name, version) + + def get_version(self, name: str, version: str) -> ModelDefinition: + return self._get_version(name, version) + + def list_versions(self, name: str) -> list[ModelVersionMeta]: + return self._list_versions(name) diff --git a/core/src/forja_core/registry/policy_store.py b/core/src/forja_core/registry/policy_store.py index c1fde9b..c0f7594 100644 --- a/core/src/forja_core/registry/policy_store.py +++ b/core/src/forja_core/registry/policy_store.py @@ -2,67 +2,24 @@ from __future__ import annotations -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import yaml - from forja_core.domain.policy import PolicyDefinition, PolicyVersionMeta -from forja_core.registry.versioning import compute_hash +from forja_core.registry.yaml_store import VersionedYamlStore -class FileSystemPolicyStore: +class FileSystemPolicyStore(VersionedYamlStore[PolicyDefinition]): """Lee políticas de policies//{index.yaml,versions/vN.yaml}.""" - def __init__(self, root: Path) -> None: - self._root = Path(root) + model_cls = PolicyDefinition + kind = "policy" def list_policies(self) -> list[PolicyDefinition]: - if not self._root.exists(): - return [] - result: list[PolicyDefinition] = [] - for d in sorted(self._root.iterdir()): - if d.is_dir() and (d / "index.yaml").exists(): - result.append(self.get_policy(d.name)) - return result + return self._list_all() def get_policy(self, name: str, version: str | None = None) -> PolicyDefinition: - index = self._load_index(name) - v = version or index["active_version"] - path = self._root / name / "versions" / f"{v}.yaml" - if not path.exists(): - raise FileNotFoundError(f"policy version not found: {name}@{v}") - with path.open(encoding="utf-8") as f: - data = yaml.safe_load(f) - return PolicyDefinition.model_validate(data) + return self._get(name, version) def list_versions(self, name: str) -> list[PolicyVersionMeta]: - index = self._load_index(name) - return [ - PolicyVersionMeta( - id=v["id"], - hash=v["hash"], - author=v["author"], - message=v["message"], - created_at=self._parse_dt(v["created_at"]), - ) - for v in index["versions"] - ] - - def _load_index(self, name: str) -> dict[str, Any]: - index_path = self._root / name / "index.yaml" - if not index_path.exists(): - raise FileNotFoundError(f"policy not found: {name}") - with index_path.open(encoding="utf-8") as f: - data: dict[str, Any] = yaml.safe_load(f) - return data - - @staticmethod - def _parse_dt(value: str | datetime) -> datetime: - if isinstance(value, datetime): - return value - return datetime.fromisoformat(value.replace("Z", "+00:00")) + return self._list_versions(name) def upsert_version( self, @@ -71,40 +28,5 @@ class FileSystemPolicyStore: message: str, author: str, ) -> PolicyVersionMeta: - policy_dir = self._root / name - versions_dir = policy_dir / "versions" - versions_dir.mkdir(parents=True, exist_ok=True) - - version_id = body.version - yaml_text = yaml.safe_dump( - body.model_dump(mode="json"), - sort_keys=False, - allow_unicode=True, - ) - h = compute_hash(yaml_text) - - version_path = versions_dir / f"{version_id}.yaml" - version_path.write_text(yaml_text, encoding="utf-8") - - meta = PolicyVersionMeta( - id=version_id, - hash=h, - author=author, - message=message, - created_at=datetime.now(UTC), - ) - - index_path = policy_dir / "index.yaml" - if index_path.exists(): - with index_path.open(encoding="utf-8") as f: - index: dict[str, Any] = yaml.safe_load(f) - else: - index = {"name": name, "versions": [], "active_version": version_id} - - index["versions"] = [v for v in index["versions"] if v["id"] != version_id] - index["versions"].append(meta.model_dump(mode="json")) - index["active_version"] = version_id - - with index_path.open("w", encoding="utf-8") as f: - yaml.safe_dump(index, f, sort_keys=False, allow_unicode=True) - return meta + # Las políticas no tienen estado draft/active: la última versión escrita manda. + return self._upsert_version(name, body, body.version, message, author, set_active=True) diff --git a/core/src/forja_core/registry/repository.py b/core/src/forja_core/registry/repository.py index 7ff7022..6adb132 100644 --- a/core/src/forja_core/registry/repository.py +++ b/core/src/forja_core/registry/repository.py @@ -3,55 +3,29 @@ from __future__ import annotations from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import yaml from forja_core.domain.agent import AgentDefinition, AgentVersionMeta -from forja_core.registry.versioning import DiffResult, compute_hash, unified_diff +from forja_core.registry.versioning import DiffResult, unified_diff +from forja_core.registry.yaml_store import VersionedYamlStore -class FileSystemAgentRegistry: +class FileSystemAgentRegistry(VersionedYamlStore[AgentDefinition]): """Lee/escribe agentes en agents//{index.yaml,versions/vN.yaml}.""" - def __init__(self, root: Path) -> None: - self._root = Path(root) + model_cls = AgentDefinition + kind = "agent" def list_agents(self) -> list[AgentDefinition]: - if not self._root.exists(): - return [] - agents: list[AgentDefinition] = [] - for d in sorted(self._root.iterdir()): - if d.is_dir() and (d / "index.yaml").exists(): - agents.append(self.get_agent(d.name)) - return agents + return self._list_all() def get_agent(self, name: str, version: str | None = None) -> AgentDefinition: - index = self._load_index(name) - v = version or index["active_version"] - return self.get_version(name, v) + return self._get(name, version) def get_version(self, name: str, version: str) -> AgentDefinition: - path = self._root / name / "versions" / f"{version}.yaml" - if not path.exists(): - raise FileNotFoundError(f"agent version not found: {name}@{version}") - with path.open(encoding="utf-8") as f: - data = yaml.safe_load(f) - return AgentDefinition.model_validate(data) + return self._get_version(name, version) def list_versions(self, name: str) -> list[AgentVersionMeta]: - index = self._load_index(name) - return [ - AgentVersionMeta( - id=v["id"], - hash=v["hash"], - author=v["author"], - message=v["message"], - created_at=self._parse_dt(v["created_at"]), - ) - for v in index["versions"] - ] + return self._list_versions(name) def upsert_version( self, @@ -60,45 +34,26 @@ class FileSystemAgentRegistry: message: str, author: str, ) -> AgentVersionMeta: - agent_dir = self._root / name - versions_dir = agent_dir / "versions" - versions_dir.mkdir(parents=True, exist_ok=True) - - version_id = body.version - yaml_text = yaml.safe_dump( - body.model_dump(mode="json"), - sort_keys=False, - allow_unicode=True, - ) - h = compute_hash(yaml_text) - - version_path = versions_dir / f"{version_id}.yaml" - version_path.write_text(yaml_text, encoding="utf-8") - - meta = AgentVersionMeta( - id=version_id, - hash=h, - author=author, - message=message, - created_at=datetime.now(UTC), + return self._upsert_version( + name, + body, + body.version, + message, + author, + set_active=body.state == "active", ) - index_path = agent_dir / "index.yaml" - if index_path.exists(): - with index_path.open(encoding="utf-8") as f: - index: dict[str, Any] = yaml.safe_load(f) - else: - index = {"name": name, "versions": [], "active_version": version_id} - - # reemplaza si existía la versión, si no añade - index["versions"] = [v for v in index["versions"] if v["id"] != version_id] - index["versions"].append(meta.model_dump(mode="json")) - if body.state == "active": - index["active_version"] = version_id - - with index_path.open("w", encoding="utf-8") as f: - yaml.safe_dump(index, f, sort_keys=False, allow_unicode=True) - return meta + def promote_to_active( + self, + name: str, + version: str, + author: str, + comment: str, + ) -> AgentVersionMeta: + """Set a version to active and update the registry index (governance promotion).""" + agent = self.get_version(name, version) + updated = agent.model_copy(update={"state": "active", "updated_at": datetime.now(UTC)}) + return self.upsert_version(name, updated, message=comment, author=author) def diff_versions(self, name: str, v1: str, v2: str) -> DiffResult: path_a = self._root / name / "versions" / f"{v1}.yaml" @@ -110,17 +65,3 @@ class FileSystemAgentRegistry: to_version=v2, unified_diff=unified_diff(text_a, text_b, f"{name}@{v1}", f"{name}@{v2}"), ) - - def _load_index(self, name: str) -> dict[str, Any]: - index_path = self._root / name / "index.yaml" - if not index_path.exists(): - raise FileNotFoundError(f"agent not found: {name}") - with index_path.open(encoding="utf-8") as f: - data: dict[str, Any] = yaml.safe_load(f) - return data - - @staticmethod - def _parse_dt(value: str | datetime) -> datetime: - if isinstance(value, datetime): - return value - return datetime.fromisoformat(str(value).replace("Z", "+00:00")) diff --git a/core/src/forja_core/registry/yaml_store.py b/core/src/forja_core/registry/yaml_store.py new file mode 100644 index 0000000..bdfb816 --- /dev/null +++ b/core/src/forja_core/registry/yaml_store.py @@ -0,0 +1,133 @@ +"""Base genérica de los registries versionados en YAML. + +Layout común en disco: ``//{index.yaml, versions/.yaml}``. +Agentes, políticas, datasets y modelos comparten esta mecánica; cada store +concreto fija ``model_cls`` (el modelo Pydantic de la definición) y ``kind`` +(el nombre del artefacto en los mensajes de error) y expone wrappers con +nombres de dominio (``get_agent``, ``get_policy``, ...). +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Generic, TypeVar + +import yaml +from pydantic import BaseModel + +from forja_core.domain.version_meta import VersionMeta +from forja_core.registry.versioning import compute_hash + +M = TypeVar("M", bound=BaseModel) + + +class VersionedYamlStore(Generic[M]): + """Operaciones de lectura sobre un árbol de artefactos versionados.""" + + model_cls: type[M] + kind: str = "item" + + def __init__(self, root: Path) -> None: + self._root = Path(root) + + def _list_all(self) -> list[M]: + if not self._root.exists(): + return [] + out: list[M] = [] + for d in sorted(self._root.iterdir()): + if d.is_dir() and (d / "index.yaml").exists(): + out.append(self._get(d.name)) + return out + + def _get(self, name: str, version: str | None = None) -> M: + index = self._load_index(name) + v: str = version or index["active_version"] + return self._get_version(name, v) + + def _get_version(self, name: str, version: str) -> M: + path = self._root / name / "versions" / f"{version}.yaml" + if not path.exists(): + raise FileNotFoundError(f"{self.kind} version not found: {name}@{version}") + with path.open(encoding="utf-8") as f: + data = yaml.safe_load(f) + return self.model_cls.model_validate(data) + + def _list_versions(self, name: str) -> list[VersionMeta]: + index = self._load_index(name) + return [ + VersionMeta( + id=v["id"], + hash=v["hash"], + author=v["author"], + message=v["message"], + created_at=self._parse_dt(v["created_at"]), + ) + for v in index["versions"] + ] + + def _load_index(self, name: str) -> dict[str, Any]: + index_path = self._root / name / "index.yaml" + if not index_path.exists(): + raise FileNotFoundError(f"{self.kind} not found: {name}") + with index_path.open(encoding="utf-8") as f: + data: dict[str, Any] = yaml.safe_load(f) + return data + + @staticmethod + def _parse_dt(value: str | datetime) -> datetime: + if isinstance(value, datetime): + return value + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + + # --- Escritura (usada por agentes y políticas) ----------------------------------- + + def _upsert_version( + self, + name: str, + body: M, + version_id: str, + message: str, + author: str, + *, + set_active: bool, + ) -> VersionMeta: + """Escribe ``versions/.yaml`` y actualiza ``index.yaml``. + + Reemplaza la entrada del índice si la versión ya existía; si + ``set_active`` es True, la marca como ``active_version``. + """ + artifact_dir = self._root / name + versions_dir = artifact_dir / "versions" + versions_dir.mkdir(parents=True, exist_ok=True) + + yaml_text = yaml.safe_dump( + body.model_dump(mode="json"), + sort_keys=False, + allow_unicode=True, + ) + (versions_dir / f"{version_id}.yaml").write_text(yaml_text, encoding="utf-8") + + meta = VersionMeta( + id=version_id, + hash=compute_hash(yaml_text), + author=author, + message=message, + created_at=datetime.now(UTC), + ) + + index_path = artifact_dir / "index.yaml" + if index_path.exists(): + with index_path.open(encoding="utf-8") as f: + index: dict[str, Any] = yaml.safe_load(f) + else: + index = {"name": name, "versions": [], "active_version": version_id} + + index["versions"] = [v for v in index["versions"] if v["id"] != version_id] + index["versions"].append(meta.model_dump(mode="json")) + if set_active: + index["active_version"] = version_id + + with index_path.open("w", encoding="utf-8") as f: + yaml.safe_dump(index, f, sort_keys=False, allow_unicode=True) + return meta diff --git a/core/src/forja_core/runtime/orchestrator.py b/core/src/forja_core/runtime/orchestrator.py index 92fb35f..bbf682f 100644 --- a/core/src/forja_core/runtime/orchestrator.py +++ b/core/src/forja_core/runtime/orchestrator.py @@ -39,9 +39,7 @@ def _thread_config(trace_id: UUID) -> dict[str, Any]: class AgentOrchestrator: """Punto único de entrada para invocar agentes, reanudar HITL y leer su estado.""" - def __init__( - self, *, provider: LLMProvider, engine: GuardrailEngine, data_dir: Path - ) -> None: + def __init__(self, *, provider: LLMProvider, engine: GuardrailEngine, data_dir: Path) -> None: self._provider = provider self._engine = engine self._data_dir = data_dir diff --git a/core/src/forja_core/training/__init__.py b/core/src/forja_core/training/__init__.py new file mode 100644 index 0000000..3c63572 --- /dev/null +++ b/core/src/forja_core/training/__init__.py @@ -0,0 +1 @@ +"""External MLOps training backends (Strategy pattern).""" diff --git a/core/src/forja_core/training/_azure_job_bundle/train.py b/core/src/forja_core/training/_azure_job_bundle/train.py new file mode 100644 index 0000000..a19a619 --- /dev/null +++ b/core/src/forja_core/training/_azure_job_bundle/train.py @@ -0,0 +1,25 @@ +"""Placeholder training script uploaded with Azure ML command jobs. + +Replace this script in your workspace or override ``AZURE_ML_COMMAND`` for real fine-tuning. +""" + +from __future__ import annotations + +import json +import os + + +def main() -> None: + payload = { + "forja_run_id": os.environ.get("FORJA_RUN_ID"), + "dataset": os.environ.get("FORJA_DATASET"), + "model": os.environ.get("FORJA_MODEL"), + "agent": os.environ.get("FORJA_AGENT"), + "learning_rate": os.environ.get("FORJA_LEARNING_RATE"), + "epochs": os.environ.get("FORJA_EPOCHS"), + } + print(json.dumps(payload)) + + +if __name__ == "__main__": + main() diff --git a/core/src/forja_core/training/azure_ml.py b/core/src/forja_core/training/azure_ml.py new file mode 100644 index 0000000..c1fddc4 --- /dev/null +++ b/core/src/forja_core/training/azure_ml.py @@ -0,0 +1,67 @@ +"""Azure ML training backend — SDK when configured, stub otherwise.""" + +from __future__ import annotations + +import asyncio +from uuid import UUID + +import structlog + +from forja_core.config import Settings +from forja_core.domain.training import ArtifactRef, BackendJobStatus, TrainingJobSpec +from forja_core.training.azure_ml_config import is_azure_ml_configured + +log = structlog.get_logger(__name__) + + +class AzureMLTrainingBackend: + """Submits command jobs via Azure ML SDK v2 when workspace settings are present.""" + + name = "azure_ml" + + def __init__(self, settings: Settings | None = None) -> None: + self._settings = settings or Settings() + self._use_sdk = is_azure_ml_configured(self._settings) + if not self._use_sdk: + log.info( + "azure_ml_stub_mode", + reason="missing workspace config, compute, or azure-ai-ml packages", + ) + + async def submit(self, spec: TrainingJobSpec, run_id: UUID) -> str: + if self._use_sdk: + from forja_core.training.azure_ml_sdk import submit_command_job + + return await asyncio.to_thread(submit_command_job, self._settings, spec, run_id) + return f"azureml-stub:{run_id}" + + async def status(self, external_job_id: str) -> BackendJobStatus: + if external_job_id.startswith("azureml-stub:"): + return _stub_status(external_job_id) + if external_job_id.startswith("azureml:"): + job_name = external_job_id.removeprefix("azureml:") + if self._use_sdk: + from forja_core.training.azure_ml_sdk import poll_job_status + + return await asyncio.to_thread(poll_job_status, self._settings, job_name) + return BackendJobStatus( + status="failed", + message="Azure ML SDK not configured; cannot poll live job", + ) + return BackendJobStatus( + status="failed", + message=f"Unknown Azure ML job id: {external_job_id}", + ) + + +def _stub_status(external_job_id: str) -> BackendJobStatus: + return BackendJobStatus( + status="succeeded", + message="Azure ML stub job completed (workspace not configured)", + artifacts=[ + ArtifactRef( + uri=f"azureml://workspaces/stub/jobs/{external_job_id}/outputs/adapter", + kind="adapter", + ), + ], + ) diff --git a/core/src/forja_core/training/azure_ml_config.py b/core/src/forja_core/training/azure_ml_config.py new file mode 100644 index 0000000..d8e5102 --- /dev/null +++ b/core/src/forja_core/training/azure_ml_config.py @@ -0,0 +1,25 @@ +"""Azure ML workspace settings helpers.""" + +from __future__ import annotations + +from forja_core.config import Settings + + +def sdk_packages_available() -> bool: + try: + import azure.ai.ml + import azure.identity # noqa: F401 + + return True + except ImportError: + return False + + +def is_azure_ml_configured(settings: Settings) -> bool: + return bool( + settings.azure_ml_subscription_id.strip() + and settings.azure_ml_resource_group.strip() + and settings.azure_ml_workspace_name.strip() + and settings.azure_ml_compute.strip() + and sdk_packages_available() + ) diff --git a/core/src/forja_core/training/azure_ml_sdk.py b/core/src/forja_core/training/azure_ml_sdk.py new file mode 100644 index 0000000..7f02e0f --- /dev/null +++ b/core/src/forja_core/training/azure_ml_sdk.py @@ -0,0 +1,100 @@ +"""Synchronous Azure ML SDK v2 client (invoked via asyncio.to_thread from the backend).""" + +from __future__ import annotations + +from pathlib import Path +from uuid import UUID + +import structlog +from azure.ai.ml import MLClient, command +from azure.identity import DefaultAzureCredential + +from forja_core.config import Settings +from forja_core.domain.training import ( + ArtifactRef, + BackendJobStatus, + TrainingJobSpec, + TrainingRunStatus, +) + +log = structlog.get_logger(__name__) + +JOB_CODE_DIR = Path(__file__).parent / "_azure_job_bundle" + +_AZURE_TO_FORJA_STATUS: dict[str, TrainingRunStatus] = { + "notstarted": "queued", + "starting": "queued", + "provisioning": "queued", + "queued": "queued", + "running": "running", + "finalizing": "running", + "completed": "succeeded", + "failed": "failed", + "canceled": "cancelled", + "cancelled": "cancelled", +} + + +def _ml_client(settings: Settings) -> MLClient: + return MLClient( + credential=DefaultAzureCredential(), + subscription_id=settings.azure_ml_subscription_id, + resource_group_name=settings.azure_ml_resource_group, + workspace_name=settings.azure_ml_workspace_name, + ) + + +def submit_command_job(settings: Settings, spec: TrainingJobSpec, run_id: UUID) -> str: + """Create an Azure ML command job; returns external id ``azureml:``.""" + client = _ml_client(settings) + display_name = f"forja-{run_id}" + env_vars = { + "FORJA_RUN_ID": str(run_id), + "FORJA_DATASET": f"{spec.dataset_name}@{spec.dataset_version}", + "FORJA_MODEL": f"{spec.model_name}@{spec.model_version}", + "FORJA_AGENT": spec.agent_name or "", + "FORJA_AGENT_VERSION": spec.agent_version or "", + "FORJA_LEARNING_RATE": str(spec.hyperparameters.learning_rate), + "FORJA_EPOCHS": str(spec.hyperparameters.epochs), + "FORJA_BATCH_SIZE": str(spec.hyperparameters.batch_size), + } + job_def = command( + code=str(JOB_CODE_DIR), + command=settings.azure_ml_command, + environment=settings.azure_ml_environment, + compute=settings.azure_ml_compute, + display_name=display_name, + experiment_name=settings.azure_ml_experiment_name, + environment_variables=env_vars, + ) + created = client.jobs.create_or_update(job_def) + job_name = created.name + if job_name is None: + raise RuntimeError("Azure ML job created without a name") + log.info("azure_ml_job_submitted", job_name=job_name, run_id=str(run_id)) + return f"azureml:{job_name}" + + +def poll_job_status(settings: Settings, job_name: str) -> BackendJobStatus: + client = _ml_client(settings) + job = client.jobs.get(job_name) + raw_status = str(job.status) if job.status is not None else "Queued" + raw = raw_status.split(".")[-1].lower() + forja_status = _AZURE_TO_FORJA_STATUS.get(raw, "running") + message = None + if forja_status == "failed": + message = "Azure ML job failed" + artifacts: list[ArtifactRef] = [] + if forja_status == "succeeded": + artifacts.append( + ArtifactRef( + uri=( + f"azureml://subscriptions/{settings.azure_ml_subscription_id}" + f"/resourcegroups/{settings.azure_ml_resource_group}" + f"/workspaces/{settings.azure_ml_workspace_name}" + f"/jobs/{job_name}/outputs/default" + ), + kind="adapter", + ) + ) + return BackendJobStatus(status=forja_status, message=message, artifacts=artifacts) diff --git a/core/src/forja_core/training/base.py b/core/src/forja_core/training/base.py new file mode 100644 index 0000000..bb145aa --- /dev/null +++ b/core/src/forja_core/training/base.py @@ -0,0 +1,22 @@ +"""TrainingBackend protocol for external fine-tuning platforms.""" + +from __future__ import annotations + +from typing import Protocol +from uuid import UUID + +from forja_core.domain.training import BackendJobStatus, TrainingJobSpec + + +class TrainingBackend(Protocol): + """Submit and poll jobs on Azure ML, SageMaker, Hugging Face Jobs, etc.""" + + name: str + + async def submit(self, spec: TrainingJobSpec, run_id: UUID) -> str: + """Start a remote job; returns provider job id.""" + ... + + async def status(self, external_job_id: str) -> BackendJobStatus: + """Poll remote job status.""" + ... diff --git a/core/src/forja_core/training/factory.py b/core/src/forja_core/training/factory.py new file mode 100644 index 0000000..4f42b2d --- /dev/null +++ b/core/src/forja_core/training/factory.py @@ -0,0 +1,14 @@ +"""Build TrainingBackend from Settings.""" + +from __future__ import annotations + +from forja_core.config import Settings +from forja_core.training.azure_ml import AzureMLTrainingBackend +from forja_core.training.base import TrainingBackend +from forja_core.training.mock import MockTrainingBackend + + +def build_training_backend(settings: Settings) -> TrainingBackend: + if settings.training_backend == "azure_ml": + return AzureMLTrainingBackend(settings) + return MockTrainingBackend() diff --git a/core/src/forja_core/training/mock.py b/core/src/forja_core/training/mock.py new file mode 100644 index 0000000..ac5524b --- /dev/null +++ b/core/src/forja_core/training/mock.py @@ -0,0 +1,28 @@ +"""Mock training backend for local development and tests.""" + +from __future__ import annotations + +from uuid import UUID + +from forja_core.domain.training import ArtifactRef, BackendJobStatus, TrainingJobSpec + + +class MockTrainingBackend: + """Immediately succeeds with a placeholder artifact URI.""" + + name = "mock" + + async def submit(self, spec: TrainingJobSpec, run_id: UUID) -> str: + return f"mock:{run_id}" + + async def status(self, external_job_id: str) -> BackendJobStatus: + return BackendJobStatus( + status="succeeded", + message="Mock training completed", + artifacts=[ + ArtifactRef( + uri=f"{external_job_id}/adapter", + kind="adapter", + ), + ], + ) diff --git a/core/src/forja_core/web/templates/_agents.html b/core/src/forja_core/web/templates/_agents.html new file mode 100644 index 0000000..1f3b20d --- /dev/null +++ b/core/src/forja_core/web/templates/_agents.html @@ -0,0 +1,79 @@ +
+ +
+ {% for card in agent_cards %} + {% set agent = card.agent %} +
+
+
+
+ {{ agent.name }} + {% for v in card.versions %} + + {{ v.id }} · {{ v.state }} + + {% endfor %} +
+

{{ agent.purpose }}

+
+
+ +
+ LLM: {{ agent.llm.provider }}/{{ agent.llm.model }} + HITL threshold: risk ≥ {{ agent.risk_threshold_for_hitl }} + Policy: + {% for g in agent.guardrails %} + {{ g }}{% if not loop.last %}, {% endif %} + {% endfor %} + + Owner: {{ agent.owner }} +
+ +
+ + Show system prompt and output schema + + +
{{ agent.system_prompt }}
+ 
{{ agent.output_schema | tojson(indent=2) }}
+
+
+ {% else %} +
No agents registered.
+ {% endfor %} +
+ +
+ {% for p in policies %} +
+
+
{{ p.name }} {{ p.version }}
+ + {{ p.on_validator_error }} + +
+

{{ p.description }}

+
+
Input validators
+
+ {% for v in p.input_validators %} + {{ v.type }} + {% endfor %} +
+
Output validators
+
+ {% for v in p.output_validators %} + {{ v.type }} + {% endfor %} +
+
+
+ {% else %} +
No policies registered.
+ {% endfor %} +
+
diff --git a/core/src/forja_core/web/templates/_approvals.html b/core/src/forja_core/web/templates/_approvals.html new file mode 100644 index 0000000..15d83ff --- /dev/null +++ b/core/src/forja_core/web/templates/_approvals.html @@ -0,0 +1,53 @@ +
+{% for ex in pending %} +
+
+
+
{{ ex.agent_name }} v{{ ex.agent_version }}
+
{{ ex.trace_id }}
+
+
Pending approval
+
+ + {% if ex.needs_human_for %} +
+
Actions awaiting approval
+ {% for a in ex.needs_human_for %} +
+
+ {{ a.action }} → {{ a.target }} + (risk={{ a.risk_score }}) +
+
Rollback: {{ a.rollback_plan }}
+
+ {% endfor %} +
+ {% endif %} + +
+ + +
+
+{% else %} +
+ No runs are waiting for approval. Run the demo incident in stage 02 — Run. +
+{% endfor %} +
diff --git a/core/src/forja_core/web/templates/_history.html b/core/src/forja_core/web/templates/_history.html new file mode 100644 index 0000000..af6c8a6 --- /dev/null +++ b/core/src/forja_core/web/templates/_history.html @@ -0,0 +1,27 @@ +
+{% for ex in runs %} +
+
+
+ {{ ex.agent_name }} + v{{ ex.agent_version }} + {{ ex.trace_id }} +
+
+ {{ ex.n_proposed_actions }} action(s) + {{ ex.n_violations }} violation(s) + Completed + {{ ex.finished_at.strftime('%H:%M:%S') if ex.finished_at else '' }} + JSON → +
+
+
+{% else %} +
+ No completed runs yet. They appear here after stage 03 — Approve (or directly when no approval is needed). +
+{% endfor %} +{% if total_completed > runs | length %} +
Showing the {{ runs | length }} most recent of {{ total_completed }} completed runs.
+{% endif %} +
diff --git a/core/src/forja_core/web/templates/_promotions.html b/core/src/forja_core/web/templates/_promotions.html new file mode 100644 index 0000000..881f792 --- /dev/null +++ b/core/src/forja_core/web/templates/_promotions.html @@ -0,0 +1,61 @@ +
+{% for item in pending %} +{% set req = item.request %} +{% set ev = item.evaluation %} +
+
+
+
{{ req.agent_name }} {{ req.agent_version }} → active
+
request {{ req.id }}
+
+
Pending promotion
+
+ +
+
+ Training run +
{{ req.training_run_id }}
+
+
+ Evaluation +
+ {% if ev %}{{ "Passed" if ev.passed else "Failed" }} ({{ ev.agent_name }}@{{ ev.agent_version }}){% else %}Unknown{% endif %} +
+
+
+ Scenarios +
{% if ev %}{{ ev.scenario_count }} run, {{ ev.violation_count }} blocking{% else %}—{% endif %}
+
+
+ +
+ Requested by {{ req.requested_by }} · {{ req.created_at.strftime('%Y-%m-%d %H:%M') if req.created_at else '' }} +
+ +
+ + +
+
+{% else %} +
+ No promotion requests are waiting. Request one from stage 04 — Train after a passing evaluation. +
+{% endfor %} +
diff --git a/core/src/forja_core/web/templates/_training.html b/core/src/forja_core/web/templates/_training.html new file mode 100644 index 0000000..1c14753 --- /dev/null +++ b/core/src/forja_core/web/templates/_training.html @@ -0,0 +1,98 @@ +
+{% for item in items %} +{% set run = item.run %} +{% set ev = item.evaluation %} +
+
+
+
+ {{ run.spec.dataset_name }}@{{ run.spec.dataset_version }} + → + {{ run.spec.model_name }}@{{ run.spec.model_version }} +
+
run {{ run.id }}
+ {% if run.spec.agent_name %} +
+ Candidate: {{ run.spec.agent_name }}{% if run.spec.agent_version %}@{{ run.spec.agent_version }}{% endif %} + {% if item.candidate_state %} + ({{ item.candidate_state }}) + {% endif %} +
+ {% endif %} +
+ + {{ run.status }} + +
+ + {% if run.error_message %} +

{{ run.error_message }}

+ {% endif %} + + {% if ev %} +
+ Evaluation + + {{ "Passed" if ev.passed else "Failed" }} + + {{ ev.scenario_count }} scenario(s), {{ ev.violation_count }} blocking violation(s) — covers {{ ev.agent_name }}@{{ ev.agent_version }} + {% if ev.notes %}— {{ ev.notes }}{% endif %} +
+ {% endif %} + +
+ {% if run.status not in ['succeeded', 'failed', 'cancelled'] %} + + {% endif %} + + {% if run.status == 'succeeded' and run.spec.agent_name and not ev %} + + Runs the candidate over its canonical scenarios. + {% endif %} + + {% if ev and ev.passed and run.spec.agent_version %} + {% if item.candidate_state == 'active' %} + ✓ Promoted — {{ run.spec.agent_name }}@{{ run.spec.agent_version }} is active (see stage 01). + {% elif item.promotion_pending %} + Promotion pending approval in stage 05 — Promote. + {% else %} + + {% endif %} + {% elif ev and ev.passed and not run.spec.agent_version %} + Set a candidate version on the run to request promotion. + {% endif %} +
+
+{% else %} +
+ No training runs yet. Submit one above — the candidate version is prefilled with the agent's draft. +
+{% endfor %} +
diff --git a/core/src/forja_core/web/templates/agent_detail.html b/core/src/forja_core/web/templates/agent_detail.html deleted file mode 100644 index 6ee81bb..0000000 --- a/core/src/forja_core/web/templates/agent_detail.html +++ /dev/null @@ -1,103 +0,0 @@ -
- -
-
- {{ agent.name }} - v{{ agent.version }} - - {{ agent.state }} - -
-
- {{ agent.purpose }} -
-
- - -
-
-
Smith
-
{{ agent.owner }}
-
-
-
Risk Threshold (Judgment)
-
{{ agent.risk_threshold_for_hitl }}
-
- {% if agent.category %} -
-
Blade Type
-
{{ agent.category }}
-
- {% endif %} -
- - -
-
-
LLM Steel
-
- {{ agent.llm.provider }} / {{ agent.llm.model }}
- temp={{ agent.llm.temperature }} · max={{ agent.llm.max_tokens }} -
-
- -
-
Guardrails (Forge Laws)
-
- {% for g in agent.guardrails %} - {{ g }} - {% endfor %} -
-
-
- - -
-
Edge & Temper
- - -
- - View blade oath - show - - -
- {{ agent.system_prompt }} -
-
- - -
-
Output schema
- 
{{ agent.output_schema | tojson(indent=2) }}
-
-
- - - {% if agent.input_label or agent.input_placeholder or agent.updated_at %} -
-
Anvil Marks
-
- {% if agent.input_label %} -
Input label: {{ agent.input_label }}
- {% endif %} - {% if agent.input_placeholder %} -
Example: {{ agent.input_placeholder }}
- {% endif %} -
Last forged: {{ agent.updated_at.strftime('%Y-%m-%d %H:%M') if agent.updated_at else '—' }}
-
-
- {% endif %} - - {% if agent.tags %} -
-
Runes
-
- {% for tag in agent.tags %} - {{ tag }} - {% endfor %} -
-
- {% endif %} -
diff --git a/core/src/forja_core/web/templates/agent_detail_full.html b/core/src/forja_core/web/templates/agent_detail_full.html deleted file mode 100644 index 18ac80b..0000000 --- a/core/src/forja_core/web/templates/agent_detail_full.html +++ /dev/null @@ -1,11 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-
- ← Back to Arsenal -
-

{{ agent.name }} v{{ agent.version }}

- {% include "agent_detail.html" %} -
-{% endblock %} diff --git a/core/src/forja_core/web/templates/agents.html b/core/src/forja_core/web/templates/agents.html deleted file mode 100644 index 2f725fc..0000000 --- a/core/src/forja_core/web/templates/agents.html +++ /dev/null @@ -1,50 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-
- ⚔ -

Arsenal

- FORGED BLADES -
- -
- -
-
- {% for a in agents %} - -
- - {{ a.icon or "◆" }} - -
-
{{ a.name }}
-
v{{ a.version }} · {{ a.state }}
-
-
- - {% else %} -
No forged blades in the Arsenal yet.
- {% endfor %} -
-
- - -
-
-
-
- 🛡 Select a blade from the Arsenal to inspect its temper, purpose and edge. -
-
-
-
-
-
-{% endblock %} diff --git a/core/src/forja_core/web/templates/approvals.html b/core/src/forja_core/web/templates/approvals.html deleted file mode 100644 index f5ca528..0000000 --- a/core/src/forja_core/web/templates/approvals.html +++ /dev/null @@ -1,51 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-
- ⚖ - JUDGMENT CHAMBER -
-

The Judgment

-

High-risk forgings pause here to be tempered or shattered. Approve or reject. The anvil waits.

- -
- {% for ex in pending %} -
-
-
-
{{ ex.agent_name }} v{{ ex.agent_version }}
-
{{ ex.trace_id }}
-
-
🛡️ AWAITING TEMPERING
-
- -
- - -
-
- {% else %} -
- 🛡️ No forgings currently await judgment. The Forge is quiet. -
- {% endfor %} -
-
-{% endblock %} diff --git a/core/src/forja_core/web/templates/armory.html b/core/src/forja_core/web/templates/armory.html deleted file mode 100644 index 6c6dfbe..0000000 --- a/core/src/forja_core/web/templates/armory.html +++ /dev/null @@ -1,61 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
- -
- ⚔ -
-

The Armory

-

The Hall of Tempered Blades — every successful forging you have completed.

-
-
- - {% if tempered %} -
- {% for ex in tempered %} -
-
-
-
{{ ex.agent_name }} v{{ ex.agent_version }}
-
{{ ex.trace_id }}
-
-
-
TEMPERED
-
- {{ ex.finished_at.strftime('%Y-%m-%d %H:%M') if ex.finished_at else ex.started_at.strftime('%Y-%m-%d %H:%M') }} -
-
-
- -
-
- ACTIONS
- {{ ex.n_proposed_actions }} -
-
- VIOLATIONS
- {{ ex.n_violations }} -
-
- -
- View full forging record → -
-
- {% endfor %} -
- {% else %} -
-
🛡️
-
No tempered blades yet
-

- Successfully complete a forging on the Anvil and it will appear here as a proven, tempered blade. -

- Go to the Anvil → -
- {% endif %} - -
-{% endblock %} \ No newline at end of file diff --git a/core/src/forja_core/web/templates/base.html b/core/src/forja_core/web/templates/base.html index 54726dd..72aa7b4 100644 --- a/core/src/forja_core/web/templates/base.html +++ b/core/src/forja_core/web/templates/base.html @@ -1,45 +1,10 @@ - + - The Forge • {{ title or "The Forge" }} - - - + Forja • {{ title or "Forja" }} + + + + + +
-
-
- - THE FORGE - Fire • Hammer • Judgment +
+ + Forja + Governed models and agents -
-
+
{% block content %}{% endblock %}
- - - {% if not hide_tutorial_button %} - - Tutorial - - {% endif %} diff --git a/core/src/forja_core/web/templates/deploy.html b/core/src/forja_core/web/templates/deploy.html deleted file mode 100644 index 8c1b3de..0000000 --- a/core/src/forja_core/web/templates/deploy.html +++ /dev/null @@ -1,48 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
- -
- ⚔ -
-

The Battle

-

Where tempered agents finally face the real world in The Battle. Send them to Azure, AWS, GCP or on-prem.

-
-
- -
- -
-
☁️ Azure
-
Deploy as Azure Functions, Container Apps, or AKS with full governance baked in.
-
Coming soon
-
- -
-
🟠 AWS
-
Lambda, ECS, EKS or SageMaker endpoints — with the same guardrails and audit trail.
-
Coming soon
-
- -
-
🔵 Google Cloud
-
Cloud Run, GKE or Vertex AI with complete decision traceability.
-
Coming soon
-
- -
-
🏠 On-Prem / Self-hosted
-
Docker, Kubernetes, or bare metal — the Forge travels with you.
-
Coming soon
-
- -
- -
- A successfully tempered blade only becomes truly valuable once it enters The Battle. - This is where your agents (with their policies, versions and full audit history) are sent into real action on Azure, AWS, GCP or on-prem — carrying the strength they earned in the Forge. -
- -
-{% endblock %} \ No newline at end of file diff --git a/core/src/forja_core/web/templates/home.html b/core/src/forja_core/web/templates/home.html deleted file mode 100644 index 91ba284..0000000 --- a/core/src/forja_core/web/templates/home.html +++ /dev/null @@ -1,40 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
- -
-

Define at The Hearth. Prove in The Crucible. Win in The Battle.

-
- -
- - - -
The Hearth
-

Define your agents: prompts, models, output schemas, risk thresholds and guardrails.

- - - - -
The Crucible
-

Put agents to the test. Run them with guardrails, handle Human-in-the-Loop decisions, and review successful runs.

- - - - -
The Battle
-

Deploy your proven agents to Azure, AWS, GCP or on-prem, ready for real action.

- - -
- - -
-
The Forge Creed
-

- The risk threshold is the critical temper point. If the blade glows white-hot, The Judgment must intervene. - Nothing leaves the Forge without passing through fire and hammer. -

-
-{% endblock %} diff --git a/core/src/forja_core/web/templates/index.html b/core/src/forja_core/web/templates/index.html new file mode 100644 index 0000000..f40d147 --- /dev/null +++ b/core/src/forja_core/web/templates/index.html @@ -0,0 +1,223 @@ +{% extends "base.html" %} + +{% macro stage_header(number, anchor, name, description) %} +
+ {{ number }} +

{{ name }}

+
+

{{ description }}

+{% endmacro %} + +{% block content %} +
+ +
+

The governed agent lifecycle, on one page.

+

+ Six stages, top to bottom. Every action below is a click — run the demo incident, + approve it, train the draft candidate, evaluate it, and promote it to active. + Each panel refreshes itself after every action. +

+
+ + + {{ stage_header("01", "define", "Define", + "Agents and the guardrail policies they run under — all versioned YAML. + The draft version below is the candidate that the rest of this page trains, + evaluates, and promotes.") }} + +
+ {% include "_agents.html" %} +
+ + +
+ {{ stage_header("02", "run", "Run", + "Execute the active agent through the governed pipeline: input guardrails → + LLM → output guardrails → proposed actions → approval gate. The demo incident + proposes a risk-5 action, so the run pauses for human approval in stage 03.") }} + +
+
+ + One click — sends a replicated HSS capacity alarm and pauses at the approval gate. +
+ +
+
+
+ + +
+
+ + +
+
+ +
+
+ +
+ + +
+ {{ stage_header("03", "approve", "Approve", + "Runs paused by the approval gate (HITL). The state lives in a checkpoint on + disk, so it survives restarts. Approving resumes the graph exactly where it + stopped; the completed run lands in stage 06.") }} + +
+ {% include "_approvals.html" %} +
+ + +
+ {{ stage_header("04", "train", "Train & evaluate", + "Submit a fine-tuning job for the draft candidate to the configured MLOps + backend, then evaluate it over the agent's canonical scenarios through the + full governed runtime. A passing evaluation unlocks promotion.") }} + +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + backend: {{ backend }} — the mock backend completes instantly. + +
+
+
+ +
+ {% include "_training.html" %} +
+ + +
+ {{ stage_header("05", "promote", "Promote", + "Governance queue. A promotion requires a succeeded training run and a passing + evaluation of the exact candidate version. Approving flips the candidate to + active in the registry — check stage 01 after approving.") }} + +
+ {% include "_promotions.html" %} +
+ + +
+ {{ stage_header("06", "audit", "Audit", + "Completed runs, persisted append-only with their full decision trail, + violations, and approved actions.") }} + +
+ {% include "_history.html" %} +
+ +
+ + +{% endblock %} diff --git a/core/src/forja_core/web/templates/run.html b/core/src/forja_core/web/templates/run.html deleted file mode 100644 index a60c02b..0000000 --- a/core/src/forja_core/web/templates/run.html +++ /dev/null @@ -1,38 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-
- ⚒ -

The Anvil

-
-

Choose a blade, dictate the task and strike. The Forge will execute with fire and rules.

- -
-
-
- - -
- -
- - -
- - -
-
- -
-
-{% endblock %} diff --git a/core/src/forja_core/web/templates/run_result.html b/core/src/forja_core/web/templates/run_result.html new file mode 100644 index 0000000..584dc8d --- /dev/null +++ b/core/src/forja_core/web/templates/run_result.html @@ -0,0 +1,68 @@ +
+
+
+ Execution result + {{ status_label }} +
+
trace {{ execution.trace_id }}
+
+ +
+
Decision path
+
+ {% for step in execution.decision_path %} +
+ {{ step.step }} + + {%- if step.duration_ms %}{{ step.duration_ms }}ms{% endif -%} + {%- if step.detail.get("n_approved") is not none %} · {{ step.detail["n_approved"] }} approved{% endif -%} + {%- if step.detail.get("rejected") %} · rejected{% endif -%} + {%- if step.detail.get("hitl") %} · HITL{% endif -%} + {%- if step.detail.get("n_violations") %} · {{ step.detail["n_violations"] }} violation(s){% endif -%} + +
+ {% else %} + — + {% endfor %} +
+
+ + {% if execution.violations %} +
+
Violations
+ {% for v in execution.violations %} +
+ • {{ v.validator }}: {{ v.message }} ({{ v.severity }}) +
+ {% endfor %} +
+ {% endif %} + + {% if execution.proposed_actions %} +
+
Proposed actions
+
+ {% for a in execution.proposed_actions %} +
+ • {{ a.action }} → {{ a.target }} + (risk={{ a.risk_score }}) +
+ {% endfor %} +
+
+ {% endif %} + + {% if execution.needs_human_for %} +
+ {{ execution.needs_human_for | length }} action(s) require approval. + Go to stage 03 — Approve ↓ +
+ {% endif %} + + {% if execution.status == "completed" %} +
+ Run finished successfully. {{ execution.proposed_actions | length }} proposed action(s). + See it in stage 06 — Audit ↓ +
+ {% endif %} +
diff --git a/core/src/forja_core/web/templates/tutorial.html b/core/src/forja_core/web/templates/tutorial.html deleted file mode 100644 index f1832a5..0000000 --- a/core/src/forja_core/web/templates/tutorial.html +++ /dev/null @@ -1,90 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
- -
-
- ✧ -

Forge Tutorial

-
-

Learn the ancient craft of forging governed agents — step by step, strike by strike.

-
- - -
-
- 01 - STATION I — THE HEARTH -
-

Visit the Arsenal

-

- Every agent in the Forge is a blade that has already been tempered. Inspect existing blades to understand their purpose, risk threshold, and the laws (guardrails) that bind them. -

- Open the Arsenal → -
- - -
-
- 02 - STATION II — THE ANVIL -
-

Strike at the Anvil

-

- This is where the real work happens. Choose a forged blade, give it a concrete task (the "steel"), and strike. The Forge will run the full pipeline: input guardrails → LLM reasoning → output guardrails → action proposal. -

-
- Try a real example. Use the incident analyzer blade with a short network incident: -
- - -
-
- -
- - -
- -
-
-
-

The result below will show the real Strike Sequence, Flaws, and Proposed Edges — exactly like a production forging.

-
- - -
-
- 03 - STATION III — THE JUDGMENT -
-

Face the Judgment (HITL)

-

- When an action's risk score crosses the blade's threshold, the forging pauses. A human (you) must bless or shatter the proposed edges before the agent can continue. This is the heart of governed autonomy. -

- Enter the Judgment Chamber → -
- - -
-
- 04 - MASTERY -
-

Repeat and Improve

-

- Every successful forging increases the overall heat of the Forge. Review history (when available), tune risk thresholds on your blades, and strengthen guardrails. True mastery comes from deliberate, observed repetition. -

-
- -
- The Forge does not forgive sloppy steel. The risk threshold is the critical temper point. If the blade glows white-hot, The Judgment must intervene. - Nothing leaves the Forge without passing through fire and hammer. -
- -
-{% endblock %} \ No newline at end of file diff --git a/core/src/forja_core/web/ui.py b/core/src/forja_core/web/ui.py index d5d9e97..b33afa6 100644 --- a/core/src/forja_core/web/ui.py +++ b/core/src/forja_core/web/ui.py @@ -1,87 +1,175 @@ -"""HTMX UI router — mounted inside forja-core (Option A).""" +"""HTMX UI router — single-page lifecycle console mounted inside forja-core. + +The UI is one page (``/``) organised as the six stages of the governed agent +lifecycle: Define → Run → Approve → Train → Promote → Audit. Each stage is a +fragment that re-fetches itself when any action button fires the global +``forja-refresh`` event, so the whole cycle can be driven with clicks only. +""" from __future__ import annotations -import asyncio +from collections.abc import Callable, Coroutine from pathlib import Path +from typing import Any +from uuid import UUID from fastapi import APIRouter, Form, Request -from fastapi.responses import HTMLResponse +from fastapi.responses import HTMLResponse, RedirectResponse from fastapi.templating import Jinja2Templates -from forja_core.api.deps import get_orchestrator, get_policy_store, get_registry, get_settings -from forja_core.api.executions import _record_execution +from forja_core.api.deps import ( + get_dataset_store, + get_model_store, + get_orchestrator, + get_policy_store, + get_registry, + get_settings, +) +from forja_core.api.evaluation_persistence import load_evaluation, load_evaluation_for_run +from forja_core.api.executions import _load_index, _record_execution from forja_core.api.persistence import append_execution, append_violation, read_execution_summaries +from forja_core.api.promotion_persistence import list_promotion_requests +from forja_core.api.training_persistence import list_training_runs from forja_core.domain.execution import AgentExecution router = APIRouter(prefix="", tags=["ui"]) -# Ruta robusta a los templates (funciona tanto en dev como dentro del contenedor) +# Template path works in dev and inside the container. TEMPLATES_DIR = Path(__file__).parent / "templates" templates = Jinja2Templates(directory=str(TEMPLATES_DIR)) -SLOGANS = [ - "Every strike, a tempered decision", - "Blades forged, not guessed", - "Fire. Hammer. Judgment.", - "The anvil never lies", - "High risk demands the Judgment", - "True steel leaves the Forge", -] +_STATUS_BADGES = { + "completed": ("text-emerald-400", "bg-emerald-950/60 border-emerald-900", "Completed"), + "awaiting_approval": ("text-amber-400", "bg-amber-950/60 border-amber-900", "Pending approval"), + "failed": ("text-red-400", "bg-red-950/60 border-red-900", "Failed"), + "blocked_by_guardrail": ("text-red-400", "bg-red-950/60 border-red-900", "Blocked by policy"), + "running": ("text-forge-steel", "bg-forge-surface2 border-forge-iron", "Running"), +} -def get_slogan() -> str: - import random - return random.choice(SLOGANS) +# Demo input for the one-click walkthrough: "hss" maps to a risk-5 action in the +# mock provider, so the run pauses in HITL and exercises the approval stage. +DEMO_INPUT = ( + "Replicated capacity alarm on the HSS active-active pair in Madrid; " + "subscriber database replication lag is growing." +) + + +# --- Context builders (shared by the index page and the fragments) --------------------- + + +def _define_ctx() -> dict[str, Any]: + """Agents with all their versions/states, plus the guardrail policies.""" + registry = get_registry() + cards = [] + for agent in registry.list_agents(): + versions = [] + for meta in registry.list_versions(agent.name): + try: + version_def = registry.get_version(agent.name, meta.id) + except FileNotFoundError: + continue + versions.append({"id": meta.id, "state": version_def.state, "message": meta.message}) + cards.append({"agent": agent, "versions": versions}) + return {"agent_cards": cards, "policies": get_policy_store().list_policies()} + + +async def _approvals_ctx() -> dict[str, Any]: + return {"pending": await _pending_executions()} + + +def _training_ctx() -> dict[str, Any]: + settings = get_settings() + registry = get_registry() + pending_promo_runs = { + str(req.training_run_id) + for req in list_promotion_requests(settings.data_dir, status="pending_approval") + } + items = [] + for run in list_training_runs(settings.data_dir): + candidate_state = None + if run.spec.agent_name and run.spec.agent_version: + try: + candidate_state = registry.get_version( + run.spec.agent_name, run.spec.agent_version + ).state + except FileNotFoundError: + candidate_state = None + items.append( + { + "run": run, + "evaluation": load_evaluation_for_run(settings.data_dir, run.id), + "candidate_state": candidate_state, + "promotion_pending": str(run.id) in pending_promo_runs, + } + ) + return { + "items": items, + "datasets": get_dataset_store().list_datasets(), + "models": get_model_store().list_models(), + "backend": settings.training_backend, + } + + +def _promotions_ctx() -> dict[str, Any]: + data_dir = get_settings().data_dir + pending_items = [] + for req in list_promotion_requests(data_dir, status="pending_approval"): + pending_items.append( + {"request": req, "evaluation": load_evaluation(data_dir, req.evaluation_report_id)} + ) + return {"pending": pending_items} + + +def _history_ctx() -> dict[str, Any]: + data_dir = get_settings().data_dir + all_execs = read_execution_summaries(data_dir) + completed = [e for e in all_execs if e.status == "completed"] + completed.sort(key=lambda e: e.finished_at or e.started_at, reverse=True) + return {"runs": completed[:10], "total_completed": len(completed)} + + +async def _pending_executions() -> list[AgentExecution]: + """Executions paused in HITL. They only live in the checkpointer (not in the + append-only JSONL, which records terminal runs), so they are reconstructed + from the execution index + a checkpointer snapshot.""" + registry = get_registry() + policy_store = get_policy_store() + orchestrator = get_orchestrator() + data_dir = get_settings().data_dir + pending: list[AgentExecution] = [] + for trace_id, meta in _load_index(data_dir).items(): + try: + agent_def = registry.get_version(meta["agent_name"], meta["version"]) + policy = policy_store.get_policy(agent_def.guardrails[0]) + execution = await orchestrator.snapshot( + agent_def=agent_def, policy=policy, trace_id=UUID(trace_id) + ) + except (FileNotFoundError, IndexError, KeyError, ValueError): + continue + if execution is not None and execution.status == "awaiting_approval": + pending.append(execution) + pending.sort(key=lambda e: e.started_at, reverse=True) + return pending + + +# --- The single page -------------------------------------------------------------------- @router.get("/", response_class=HTMLResponse) -async def home(request: Request) -> HTMLResponse: - return templates.TemplateResponse( - "home.html", - {"request": request, "title": "Home", "slogan": get_slogan()}, - ) - - -@router.get("/agents", response_class=HTMLResponse) -async def agents_page(request: Request) -> HTMLResponse: +async def index(request: Request) -> HTMLResponse: registry = get_registry() - agents = registry.list_agents() - return templates.TemplateResponse( - "agents.html", - {"request": request, "title": "Agents", "agents": agents, "slogan": get_slogan()}, - ) - - -@router.get("/agents/{name}", response_class=HTMLResponse) -async def agent_detail(request: Request, name: str) -> HTMLResponse: - registry = get_registry() - try: - agent = registry.get_agent(name) - except FileNotFoundError: - return HTMLResponse('
Blade not found in the Arsenal
', status_code=404) - - # Si viene por HTMX, devolvemos solo el fragmento (sin layout) - if request.headers.get("HX-Request"): - return templates.TemplateResponse( - "agent_detail.html", - {"request": request, "agent": agent, "slogan": get_slogan()}, - ) - - # Acceso directo por navegador → página completa - return templates.TemplateResponse( - "agent_detail_full.html", - {"request": request, "title": agent.name, "agent": agent, "slogan": get_slogan()}, - ) - - -@router.get("/run", response_class=HTMLResponse) -async def run_page(request: Request) -> HTMLResponse: - registry = get_registry() - agents = registry.list_agents() - return templates.TemplateResponse( - "run.html", - {"request": request, "title": "Run", "agents": agents, "slogan": get_slogan()}, - ) + ctx: dict[str, Any] = { + "request": request, + "title": "Lifecycle", + "agents": registry.list_agents(), + "demo_input": DEMO_INPUT, + } + ctx.update(_define_ctx()) + ctx.update(await _approvals_ctx()) + ctx.update(_training_ctx()) + ctx.update(_promotions_ctx()) + ctx.update(_history_ctx()) + return templates.TemplateResponse("index.html", ctx) @router.post("/run", response_class=HTMLResponse) @@ -91,143 +179,116 @@ async def run_invoke( input: str = Form(...), ) -> HTMLResponse: """Execute the agent using the orchestrator directly (same process).""" - from forja_core.api.deps import get_policy_store - registry = get_registry() policy_store = get_policy_store() orchestrator = get_orchestrator() settings = get_settings() try: - agent_def = registry.get_agent(agent_name) # resuelve la versión activa + agent_def = registry.get_agent(agent_name) policy_name = agent_def.guardrails[0] if agent_def.guardrails else "default" policy = policy_store.get_policy(policy_name) - execution: AgentExecution = asyncio.run( - orchestrator.invoke( - agent_def=agent_def, - policy=policy, - user_input=input, - ) + execution: AgentExecution = await orchestrator.invoke( + agent_def=agent_def, + policy=policy, + user_input=input, ) - # Persist terminal forgings (including successful "BLADE TEMPERED" ones) - # so they appear in the Armory / Vault of Tempered Blades. - _record_execution(settings.data_dir, str(execution.trace_id), agent_def.name, agent_def.version) + # Persist like the API does: index always, JSONL only for terminal runs. + _record_execution( + settings.data_dir, str(execution.trace_id), agent_def.name, agent_def.version + ) if execution.status in {"completed", "failed", "blocked_by_guardrail"}: append_execution(settings.data_dir, execution) - for v in execution.violations: - append_violation(settings.data_dir, v) + for v in execution.violations: + append_violation(settings.data_dir, v) except Exception as e: - return HTMLResponse(f'
Forge error: {str(e)[:300]}
', status_code=400) + return HTMLResponse( + f'
' + f"Execution error: {str(e)[:300]}
", + status_code=400, + ) - # Forging result (gamified) - status_map = { - "completed": ("text-emerald-400", "bg-emerald-950/60 border-emerald-900", "BLADE TEMPERED"), - "awaiting_approval": ("text-amber-400", "bg-amber-950/60 border-amber-900", "ON THE ANVIL · JUDGMENT REQUIRED"), - "failed": ("text-red-400", "bg-red-950/60 border-red-900", "SHATTERED IN THE FIRE"), - "blocked_by_guardrail": ("text-red-400", "bg-red-950/60 border-red-900", "BLOCKED BY THE FORGE"), - } - text_color, badge_style, status_label = status_map.get( - execution.status, ("text-forge-steel", "bg-forge-surface2 border-forge-iron", execution.status.upper()) + text_color, badge_style, status_label = _STATUS_BADGES.get( + execution.status, + ( + "text-forge-steel", + "bg-forge-surface2 border-forge-iron", + execution.status.replace("_", " ").title(), + ), ) - - steps_html = "" - for step in execution.decision_path: - dur = f"{step.duration_ms}ms" if step.duration_ms else "" - extra = "" - if step.n_approved is not None: - extra = f" · {step.n_approved} blessings" - if step.rejected: - extra = " · SHATTERED" - if step.hitl: - extra = " · JUDGMENT" - steps_html += f'
{step.node}{dur}{extra}
' - - violations_html = "" - if execution.violations: - for v in execution.violations: - sev = "text-red-400" if v.severity == "block" else "text-amber-400" - violations_html += f'
• {v.validator}: {v.message} ({v.severity})
' - - actions_html = "" - if execution.proposed_actions: - for a in execution.proposed_actions: - risk = a.risk_score - badge = "text-emerald-400" if risk < 3 else ("text-amber-400" if risk < 5 else "text-red-400") - actions_html += f'
• {a.description} (risk={risk})
' - - needs_html = "" - if execution.needs_human_for: - needs_html = f'
This blade awaits the Judgment ({len(execution.needs_human_for)} action(s)). Enter the Chamber →
' - - html = f""" -
-
-
- Forging Record - {status_label} -
-
trace {execution.trace_id}
-
- -
-
Strike Sequence
-
{steps_html or 'no strikes'}
-
- - {f'
Flaws (Violations)
{violations_html}
' if violations_html else ''} - - {f'
Proposed Edges
{actions_html}
' if actions_html else ''} - - {needs_html} - - {f'
✓ Forging complete. {len(execution.proposed_actions or [])} edges ready.
' if execution.status == 'completed' else ''} -
- """ - return HTMLResponse(html) - - -# ====================== APPROVALS (HITL) ====================== - -@router.get("/tutorial", response_class=HTMLResponse) -async def tutorial_page(request: Request) -> HTMLResponse: - """Gamified step-by-step guide that teaches how to forge an agent.""" return templates.TemplateResponse( - "tutorial.html", - {"request": request, "title": "Tutorial", "slogan": get_slogan(), "hide_tutorial_button": True}, + "run_result.html", + { + "request": request, + "execution": execution, + "status_label": status_label, + "status_text_color": text_color, + "status_badge_style": badge_style, + }, ) -@router.get("/armory", response_class=HTMLResponse) -async def armory_page(request: Request) -> HTMLResponse: - """The Armory — collection of successfully tempered blades (completed forgings).""" - data_dir = get_settings().data_dir - all_execs = read_execution_summaries(data_dir) - tempered = [e for e in all_execs if e.status == "completed"] - tempered.sort(key=lambda e: e.finished_at or e.started_at, reverse=True) +# --- Fragments (re-fetched on the global forja-refresh event) --------------------------- + + +@router.get("/fragments/agents", response_class=HTMLResponse) +async def fragment_agents(request: Request) -> HTMLResponse: + return templates.TemplateResponse("_agents.html", {"request": request, **_define_ctx()}) + + +@router.get("/fragments/approvals", response_class=HTMLResponse) +async def fragment_approvals(request: Request) -> HTMLResponse: return templates.TemplateResponse( - "armory.html", - {"request": request, "title": "Armory", "tempered": tempered, "slogan": get_slogan()}, + "_approvals.html", {"request": request, **(await _approvals_ctx())} ) -@router.get("/deploy", response_class=HTMLResponse) -async def deploy_page(request: Request) -> HTMLResponse: - """Deployment station — where tempered agents are sent to Azure, AWS, GCP, etc.""" - return templates.TemplateResponse( - "deploy.html", - {"request": request, "title": "Deploy", "slogan": get_slogan()}, - ) +@router.get("/fragments/training", response_class=HTMLResponse) +async def fragment_training(request: Request) -> HTMLResponse: + return templates.TemplateResponse("_training.html", {"request": request, **_training_ctx()}) -@router.get("/approvals", response_class=HTMLResponse) -async def approvals_page(request: Request) -> HTMLResponse: - data_dir = get_settings().data_dir - all_execs = read_execution_summaries(data_dir) - pending = [e for e in all_execs if e.status == "awaiting_approval"] - return templates.TemplateResponse( - "approvals.html", - {"request": request, "title": "Approvals", "pending": pending, "slogan": get_slogan()}, +@router.get("/fragments/promotions", response_class=HTMLResponse) +async def fragment_promotions(request: Request) -> HTMLResponse: + return templates.TemplateResponse("_promotions.html", {"request": request, **_promotions_ctx()}) + + +@router.get("/fragments/history", response_class=HTMLResponse) +async def fragment_history(request: Request) -> HTMLResponse: + return templates.TemplateResponse("_history.html", {"request": request, **_history_ctx()}) + + +# --- Legacy routes from the multi-page UI → anchors on the single page ------------------ + +_LEGACY_ROUTES = { + "/agents": "/#define", + "/policies": "/#define", + "/run": "/#run", + "/approvals": "/#approve", + "/training": "/#train", + "/promotions": "/#promote", + "/history": "/#audit", + "/armory": "/#audit", + "/tutorial": "/", +} + + +def _redirect_endpoint(target: str) -> Callable[[], Coroutine[Any, Any, RedirectResponse]]: + async def _redirect() -> RedirectResponse: + return RedirectResponse(url=target, status_code=301) + + return _redirect + + +for _path, _target in _LEGACY_ROUTES.items(): + router.add_api_route( + _path, _redirect_endpoint(_target), methods=["GET"], include_in_schema=False ) + +# Old per-agent detail pages → Define stage. +router.add_api_route( + "/agents/{name}", _redirect_endpoint("/#define"), methods=["GET"], include_in_schema=False +) diff --git a/datasets/incident_sft/index.yaml b/datasets/incident_sft/index.yaml new file mode 100644 index 0000000..01bb445 --- /dev/null +++ b/datasets/incident_sft/index.yaml @@ -0,0 +1,8 @@ +name: incident_sft +versions: + - id: v1 + hash: pending + author: Juan + message: SFT examples derived from incident_analyzer scenarios. + created_at: 2026-06-01T10:00:00Z +active_version: v1 \ No newline at end of file diff --git a/datasets/incident_sft/records/v1.jsonl b/datasets/incident_sft/records/v1.jsonl new file mode 100644 index 0000000..d0cfd92 --- /dev/null +++ b/datasets/incident_sft/records/v1.jsonl @@ -0,0 +1 @@ +{"input": "SIP registration drop after image promotion", "output": {"severity": "high"}} \ No newline at end of file diff --git a/datasets/incident_sft/versions/v1.yaml b/datasets/incident_sft/versions/v1.yaml new file mode 100644 index 0000000..ba47c3d --- /dev/null +++ b/datasets/incident_sft/versions/v1.yaml @@ -0,0 +1,8 @@ +name: incident_sft +version: v1 +owner: Juan +purpose: Supervised fine-tuning records for voice platform incident analysis. +state: active +format: jsonl +records_path: records/v1.jsonl +updated_at: 2026-06-01T10:00:00Z \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 72655ec..01251a1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,9 +13,15 @@ services: DATA_DIR: /app/data AGENTS_DIR: /app/agents POLICIES_DIR: /app/policies + DATASETS_DIR: /app/datasets + MODELS_DIR: /app/models volumes: + # Mount source so local UI/API changes apply without rebuilding the image. + - ./core/src:/app/src:ro - ./agents:/app/agents:rw - ./policies:/app/policies:rw + - ./datasets:/app/datasets:ro + - ./models:/app/models:ro - ./data:/app/data healthcheck: test: ["CMD", "curl", "-fsS", "http://localhost:8000/health"] diff --git a/docs/componentes.md b/docs/componentes.md index 0f92a13..84057b2 100644 --- a/docs/componentes.md +++ b/docs/componentes.md @@ -332,12 +332,14 @@ Hashing/versionado: `registry/versioning.compute_hash(yaml_text)` = SHA-256 del ## 6. UI HTMX embebida en Core -Con la migración a Opción A (FastAPI + HTMX), la interfaz de usuario se sirve -directamente desde el mismo proceso `forja-core` (Jinja2 + HTMX vía CDN, sin -servicio separado). No hay `CoreClient` intermedio para las páginas; los -endpoints de UI llaman a los mismos servicios internos que la API REST. - -La antigua sección de dashboard Streamlit ha sido eliminada. +La interfaz es **una sola página** (`GET /`) servida desde el mismo proceso +`forja-core` (Jinja2 + HTMX vía CDN, sin servicio separado), organizada como +las seis etapas del ciclo de vida: Define → Run → Approve → Train → Promote → +Audit. Cada etapa es un fragmento (`GET /fragments/*`) que se re-renderiza +cuando cualquier botón de acción dispara el evento global `forja-refresh`. +Los botones llaman directamente a los endpoints `/api/*` (con `json-enc`); +no hay cliente HTTP intermedio ni lógica duplicada. Las rutas multipágina +antiguas redirigen con 301 a las anclas de su etapa. --- @@ -366,22 +368,28 @@ env `DATA_DIR=/app/data`, `AGENTS_DIR=/app/agents`, `POLICIES_DIR=/app/policies` | Campo | Env var | Default | Lo consume | |-------|---------|---------|------------| | `llm_provider` | `LLM_PROVIDER` | `mock` | `build_llm_provider` | -| `llm_fallback_provider` | `LLM_FALLBACK_PROVIDER` | `""` | (declarado; la factory aún no lo usa) | +| `llm_fallback_provider` | `LLM_FALLBACK_PROVIDER` | `""` | `build_llm_provider` → `FallbackLLMProvider` (vacío = sin fallback) | | `azure_openai_*` | `AZURE_OPENAI_*` | `""` / `2024-08-01-preview` | `AzureOpenAIProvider` | | `openai_api_key` / `openai_model` | `OPENAI_API_KEY` / `OPENAI_MODEL` | `""` / `gpt-4o` | `OpenAIProvider` | | `guardrails_nemo_enabled` | `GUARDRAILS_NEMO_ENABLED` | `False` | `build_guardrail_engine` | +| `guardrails_nemo_allowed_keywords` | `GUARDRAILS_NEMO_ALLOWED_KEYWORDS` | `""` | keywords CSV del stub topical-rails (vacío = sin chequeo) | | `log_level` | `LOG_LEVEL` | `INFO` | `configure_logging` | -| `data_dir` | `DATA_DIR` | `./data` | `build_checkpointer`, `_record_execution`, `append_*`, `read_*` | -| `agents_dir` | `AGENTS_DIR` | `./agents` | `build_agent_registry` | +| `training_backend` | `TRAINING_BACKEND` | `mock` | `build_training_backend` | +| `azure_ml_*` | `AZURE_ML_*` | varios | `AzureMLTrainingBackend` (stub si falta workspace) | +| `data_dir` | `DATA_DIR` | `./data` | checkpointer, logs JSONL, training runs, evaluaciones, promociones | +| `agents_dir` | `AGENTS_DIR` | `./agents` | `build_agent_registry`, escenarios de evaluación | | `policies_dir` | `POLICIES_DIR` | `./policies` | `build_policy_store` | +| `datasets_dir` | `DATASETS_DIR` | `./datasets` | `build_dataset_store` | +| `models_dir` | `MODELS_DIR` | `./models` | `build_model_store` | --- ## 8. Aristas y "gotchas" -- **`llm_fallback_provider`**: existe en `Settings` y en `.env.example`, pero - `build_llm_provider` aún no lo aplica (queda como punto de extensión). -- **`build_llm_provider`** usa `match` sin `case _:`; al ser el tipo un `Literal` +- **`llm_fallback_provider`**: si está configurado y difiere del primario, + `build_llm_provider` devuelve un `FallbackLLMProvider` que delega en el + secundario cuando el primario agota sus reintentos. +- **`_build_single`** usa `match` sin `case _:`; al ser el tipo un `Literal` de tres valores es exhaustivo, pero un valor inesperado caería en "ninguna rama". - **NeMo**: `NeMoGuardrailsEngine` solo entra al `CompositeGuardrailEngine` si `GUARDRAILS_NEMO_ENABLED=true`; por defecto el composite tiene un solo engine. diff --git a/docs/explicacion.md b/docs/explicacion.md index 36d1e78..6615e2d 100644 --- a/docs/explicacion.md +++ b/docs/explicacion.md @@ -1,9 +1,9 @@ # Forja explicado de principio a fin -> **Nota de estado (mayo 2026):** Este documento fue escrito para la arquitectura -> anterior (dos servicios + Streamlit/NiceGUI). Forja ahora es un **único servicio -> FastAPI** con UI HTMX embebida. El núcleo (runtime, guardrails, HITL, versionado) -> sigue siendo el mismo. Algunas secciones de UI y despliegue están desactualizadas. +> **Nota de estado (junio 2026):** Forja es un **único servicio FastAPI** con UI +> HTMX embebida. Además del runtime gobernado, incluye la capa MLOps: datasets y +> modelos versionados, training runs contra backends externos, evaluación +> post-training y promoción gobernada draft → active. > > Si solo quieres arrancarlo, ve al [`README.md`](../README.md). @@ -49,37 +49,33 @@ Si entiendes estas seis ideas, entiendes el proyecto. Todo lo demás son detalle --- -## 3. Vista de pájaro: dos servicios +## 3. Vista de pájaro: un único servicio -Forja son **dos procesos** que se hablan por HTTP/JSON: +Forja es **un solo proceso** FastAPI (puerto 8000) con dos caras: ``` ┌───────────────────────────── docker-compose ──────────────────────────────┐ -│ │ -│ ┌──────────────────────┐ HTTP/JSON ┌────────────────────────┐ │ -│ │ forja-dashboard │ ───────────────► │ forja-core │ │ -│ │ Streamlit :8501 │ ◄─────────────── │ FastAPI :8000 │ │ -│ │ (la "consola") │ │ (el cerebro) │ │ -│ └──────────────────────┘ └───────────┬────────────┘ │ -│ │ │ -│ ┌────────────────┬────────────────────┼───────────────┤ -│ ▼ ▼ ▼ │ -│ capa LLM capa Guardrails runtime LangGraph │ -│ (Strategy+factory) (Strategy+factory) (grafo + checkpointer) │ -│ │ │ │ │ -│ └────────────────┴──────────┬──────────┘ │ -│ ▼ │ -│ Persistencia: YAML · JSON · JSONL · SQLite │ +│ forja-core :8000 │ +│ ┌─────────────────────────────────────────────────────────────────┐ │ +│ │ UI HTMX (humanos, Jinja2) │ API REST (/api/*) │ │ +│ └────────────────────────────┬────────────────────────────────────┘ │ +│ ▼ │ +│ capa LLM capa Guardrails runtime LangGraph │ +│ (Strategy+factory) (Strategy+factory) (grafo + checkpointer) │ +│ │ │ │ │ +│ └──────────────────┴───────────┬──────────┘ │ +│ ▼ │ +│ Persistencia: YAML · JSON · JSONL · SQLite │ └────────────────────────────────────────────────────────────────────────────┘ ``` -- **`forja-core`** (FastAPI, puerto 8000) — todo el dominio: registry de - agentes, motor de guardrails, runtime de ejecución, persistencia. No tiene UI. -- **`forja-dashboard`** (Streamlit, puerto 8501) — una consola visual. **No - contiene lógica de negocio**: es un cliente HTTP del core con cinco páginas. - -La separación importa: el core podría servir a una CLI, a otro servicio, a un -pipeline... el dashboard es solo una de las caras posibles. +- La **API REST** (bajo `/api`) es el contrato para máquinas: CLI, pipelines, + otros servicios. +- La **UI HTMX** es **una sola página** (`/`) organizada como las seis etapas + del ciclo de vida (Define → Run → Approve → Train → Promote → Audit). Se + sirve desde el mismo proceso y llama a los mismos servicios internos — no hay + un cliente HTTP intermedio ni lógica de negocio duplicada. Las rutas antiguas + (`/agents`, `/run`, ...) redirigen a su etapa. ### Las capas del core (de fuera hacia dentro) @@ -285,25 +281,20 @@ inyectan). Dependen de él: la API (`api/executions.py` solo conoce el Y la **raíz de la app**: `core/src/ forja_core/main.py` → `create_app()` instancia `FastAPI`, añade el `TraceIdMiddleware`, monta los routers, y expone `/health`. -Depende de: todo lo de arriba (vía `deps.py`). Dependen de él: el dashboard (por -HTTP) y los tests de integración (`tests/integration/`, vía `TestClient`). +Depende de: todo lo de arriba (vía `deps.py`). Dependen de él: la UI HTMX +(mismo proceso) y los tests (`TestClient`). -### 4.8 `dashboard/` — la consola Streamlit +### 4.8 `web/` — la UI HTMX embebida (una página) | Fichero | Rol | |---------|-----| -| `client.py` | `CoreClient`: un cliente HTTP síncrono (httpx, con reintentos) que envuelve **todos** los endpoints del core y mapea 404/409/422 a `{"error": ...}`. Es lo único que sabe hablar con el core. | -| `app.py` | La página raíz: sidebar de branding + un health-check del core. | -| `pages/1_🏛️_Registro.py` | Catálogo de agentes: detalle (prompt, esquema, LLM, guardrails), tabla de versiones, **diff coloreado v1↔v2**. | -| `pages/2_▶️_Ejecutar.py` | Lanza un agente: botones con los escenarios pregrabados, textarea, `invoke`, y render del status + output + violaciones + *timeline* del `decision_path`. Avisa si quedó en `awaiting_approval`. | -| `pages/3_🤝_Aprobaciones.py` | La cola de HITL: lista las ejecuciones `awaiting_approval`, muestra cada acción propuesta (risk_score coloreado, target, rollback_plan) con un checkbox, y aprueba el subconjunto elegido o rechaza con motivo. | -| `pages/4_📜_Historial.py` | Pestaña de ejecuciones (tabla + detalle por `trace_id`) y pestaña de violaciones (filtrable por severidad). | -| `pages/5_📐_Politicas.py` | Inventario de políticas: validadores de entrada/salida (cada uno expandible con su config) y versiones. | -| `components/` | Trozos reutilizables de UI: `diff_view` (pinta `+`/`-`/`@@`), `trace_view` (el timeline del `decision_path`), `violation_view` (badges de severidad). | +| `ui.py` | `GET /` (la página completa), `POST /run`, `GET /fragments/{agents,approvals,training,promotions,history}` y redirects 301 de las rutas antiguas. Llama a los mismos singletons de `deps.py` que la API; no hay cliente HTTP intermedio. | +| `templates/index.html` | La página única: seis etapas (Define → Run → Approve → Train → Promote → Audit), cada una con su explicación y su panel. | +| `templates/_*.html` | Fragmentos por etapa. Cada panel lleva `hx-get="/fragments/X" hx-trigger="forja-refresh from:body"`: cualquier botón de acción dispara el evento `forja-refresh` al terminar y **todos los paneles se re-renderizan** — el ciclo entero se recorre con clicks, sin recargar la página. | -Depende de: el `forja-core` por HTTP (vía `FORJA_CORE_URL`). **Nadie del -core depende del dashboard.** No tiene tests unitarios (mal coste/beneficio para -Streamlit); su verificación es la checklist manual de [`docs/manual_qa.md`](manual_qa.md). +Los botones de acción (approve/reject/train/evaluate/promote) hacen `hx-post` +con la extensión `json-enc` directamente contra los endpoints `/api/*`; el botón +**Run demo incident** lanza un incidente pregrabado que pausa en HITL. ### 4.9 Lo que no es código: `agents/`, `policies/`, `data/` @@ -348,7 +339,7 @@ Esta es la sección que conviene leer despacio: aquí se ve cómo encaja todo. S ### Acto 1 — la petición entra y se ejecuta el grafo ``` -Cliente (dashboard o curl) +Cliente (UI o curl) │ POST /agents/incident_analyzer/invoke { "input": "" } ▼ TraceIdMiddleware ............ genera trace_id = UUID, lo bind-ea al log @@ -391,10 +382,10 @@ router: status no es terminal → NO se escribe en executions.jsonl, respuesta 200 { status: "awaiting_approval", trace_id, needs_human_for: [act-1], decision_path: [...], ... } ``` -En el dashboard, la página **Ejecutar** muestra el timeline y un aviso "ve a -Aprobaciones". La página **Aprobaciones** hace `GET /executions`, encuentra esta -ejecución (el endpoint reconstruye las `awaiting_approval` desde el índice + el -checkpointer) y muestra `act-1` con su risk_score, target y rollback_plan. +En la UI, la página **Runs** muestra el timeline y un aviso "Open Approvals". +La página **Approvals** reconstruye las ejecuciones `awaiting_approval` desde el +índice + el checkpointer y muestra `act-1` con su risk_score, target y +rollback_plan. ### Acto 2 — el humano decide; el grafo se reanuda @@ -431,7 +422,7 @@ respuesta 200 { status: "completed", final_output: { ..., approved_actions: [ac ```mermaid sequenceDiagram - participant U as Cliente / Dashboard + participant U as Cliente / UI participant MW as TraceIdMiddleware participant API as router (api/executions.py) participant ORCH as AgentOrchestrator @@ -477,6 +468,7 @@ Forja no usa una sola base de datos; usa la herramienta adecuada para cada cosa. | Mapa `trace_id → agente/versión` | **JSON** (`execution_index.json`) | Necesario para reanudar un HITL sabiendo qué configuración lo ejecutó; debe sobrevivir a reinicios. | | Log de ejecuciones terminales y de violaciones | **JSONL append-only** | Inmutable, auditable, trivial de "shipear" a un sistema de logs. | | Estado intermedio del grafo (incl. pausas HITL) | **SQLite** (`checkpoints.sqlite`, vía LangGraph) | Es lo que LangGraph espera; permite reanudar una ejecución pausada entre reinicios del proceso. | +| Training runs, evaluaciones y promociones | **JSON por entidad** bajo `data/` (+ JSONL para promociones aprobadas) | Estado mutable consultable por id; la auditoría de promociones es append-only. | --- @@ -497,6 +489,9 @@ Forja no usa una sola base de datos; usa la herramienta adecuada para cada cosa. | **Factory** | Función `build_X(settings)` que monta la implementación de `X` adecuada según la configuración. | | **Status de ejecución** | `running`, `awaiting_approval` (pausada en HITL), `blocked_by_guardrail` (la frenó un validador), `completed`, `failed`. | | **Estados de un agente** | `draft`, `active`, `deprecated` (metadato en su YAML). | +| **Training run** | Job de fine-tuning enviado a un backend externo (mock o Azure ML) con dataset, modelo base y versión candidata del agente. | +| **Evaluación post-training** | Ejecutar los escenarios canónicos del agente (`examples/*.txt`) por el runtime gobernado; pasa si no hay violaciones bloqueantes. Es el gate de promoción. | +| **Promoción** | Petición de gobernanza para pasar una versión `draft` a `active`; requiere run `succeeded` + evaluación `passed` de esa misma versión + aprobación humana. | --- @@ -508,18 +503,22 @@ Forja no usa una sola base de datos; usa la herramienta adecuada para cada cosa. │ ├── src/ forja_core/ │ │ ├── domain/ ← los modelos de datos (empieza por execution.py) │ │ ├── config.py · observability/ ← cimientos transversales -│ │ ├── llm/ ← proveedores LLM (Protocol + impls + factory) -│ │ ├── registry/ ← catálogo y versionado (YAML/JSON, hash, diff) +│ │ ├── llm/ ← proveedores LLM (Protocol + impls + fallback + factory) +│ │ ├── registry/ ← catálogo y versionado (base yaml_store.py + 4 stores) │ │ ├── guardrails/ ← motor de validación (Protocol + composite + validators) │ │ ├── runtime/ ← el grafo LangGraph (state, nodes, graph, checkpointer, orchestrator) +│ │ ├── training/ ← backends de fine-tuning (mock, Azure ML) +│ │ ├── evaluation/ ← escenarios canónicos + evaluación post-training │ │ ├── api/ ← FastAPI (middlewares, deps, routers, persistence) +│ │ ├── web/ ← UI HTMX (ui.py + templates/) │ │ └── main.py ← create_app(): ensambla la app │ ├── Dockerfile · requirements.txt ├── agents/incident_analyzer/ ← el agente de ejemplo (YAMLs + escenarios .txt) ├── policies/default/ ← la política de guardrails de ejemplo +├── datasets/ · models/ ← datasets y modelos base versionados (YAML) ├── data/ ← estado runtime (gitignored) ├── tests/ ← pytest: tests/unit/ y tests/integration/ -├── docs/ ← este documento, manual_qa.md, futuro.md, superpowers/ +├── docs/ ← este documento, componentes.md, futuro.md, product-voice.md ├── docker-compose.yml ← levanta core (API + UI HTMX en puerto 8000) ├── Makefile ← install / test / test-all / lint / smoke / up / down ├── README.md · ARCHITECTURE.md @@ -531,7 +530,8 @@ Forja no usa una sola base de datos; usa la herramienta adecuada para cada cosa. 3. `runtime/nodes.py` y `runtime/graph.py` — el flujo de ejecución. 4. `api/executions.py` — cómo se expone (invoke / approve / reject). 5. `tests/integration/test_invoke_hitl.py` — el ciclo completo, en ~40 líneas. -6. Levanta `docker compose up` y haz clic por las cinco páginas del dashboard. +6. Levanta `docker compose up` y recorre la página única de arriba abajo: + los 6 clicks de la demo del README cubren el ciclo completo. --- @@ -540,4 +540,4 @@ Forja no usa una sola base de datos; usa la herramienta adecuada para cada cosa. Es un MVP. La detección de PII más fina (modelos grandes, español), autenticación, OpenTelemetry, multi-tenant, persistencia en Postgres, evaluadores LLM-as-judge, una cola de aprobaciones con SLA... están en el roadmap: [`docs/futuro.md`](futuro.md). -El estado actual y cómo verificarlo: [`README.md`](../README.md) y [`docs/manual_qa.md`](manual_qa.md). +El estado actual y cómo verificarlo: [`README.md`](../README.md). diff --git a/docs/forja-progress.html b/docs/forja-progress.html deleted file mode 100644 index 7bf8c4c..0000000 --- a/docs/forja-progress.html +++ /dev/null @@ -1,124 +0,0 @@ - - - - - - Forja — Estado Actual - - - - -
- - -
- ⚒︎ -
-

Forja

-

Estado actual del proyecto (Gamificado)

-
-
- -
-
- - Versión actual: 3 estaciones -
-
- - -
-

Resumen del Estado Actual

- -
-

- El proyecto ha sido simplificado a 3 estaciones principales, siguiendo una estructura clara y lógica: -

- -
- -
-
STATION I
-
The Hearth
-
Definición de agentes (Arsenal)
-
- - -
-
STATION II
-
The Crucible
-
Pruebas, guardrails y Human-in-the-Loop (Agrupación de Anvil + Judgment + Armory)
-
- - -
-
STATION III
-
The Battle
-
Despliegue de agentes forjados (Azure, AWS, GCP, On-prem)
-
-
- -

Cambios principales realizados:

- -
    -
  • ✓ Reducción de 5 estaciones a 3 estaciones bien definidas.
    • -
  • ✓ Eliminación de etiquetas redundantes ("STATION I", "STATION II"...).
    • -
  • ✓ Títulos de estaciones ahora en naranja (ember) y tamaño grande.
    • -
  • ✓ Texto guía superior en blanco y tamaño grande (text-3xl).
    • -
  • ✓ Recuperación del Forge Creed original como elemento inspirador.
    • -
  • ✓ Eliminación de los enlaces "Go to the..." de las tarjetas para mayor limpieza.
    • -
  • ✓ Botón flotante de Tutorial simplificado y centrado.
    • -
  • ✓ Navbar con slogan: "Fire • Hammer • Judgment"
    • -
  • ✓ The Armory ya no es una estación principal (se integra conceptualmente dentro de The Crucible).
    • -
-
-
- -
-

Próximos pasos sugeridos:

-
    -
  • • Mejorar la página de The Battle (/deploy) con más contenido real.
    • -
  • • Conectar visualmente The Crucible con las páginas de ejecución y aprobaciones.
    • -
  • • Añadir feedback visual de "éxito" cuando un agente se forja correctamente.
    • -
  • • Posiblemente añadir un estado de "agentes desplegados" en The Battle.
    • -
-
- -
- - \ No newline at end of file diff --git a/docs/futuro.md b/docs/futuro.md index d300cec..af421b7 100644 --- a/docs/futuro.md +++ b/docs/futuro.md @@ -1,15 +1,30 @@ # Roadmap (post-MVP) -- **NeMo Guardrails full**: configuración Colang con KB embedding + dialog rails. -- **Autenticación**: OAuth2/OIDC con MSAL para entornos Azure AD. +- **Training real**: conectar los jobs de Azure ML a scripts de fine-tuning + reales sobre los datasets registrados; regression LLM-as-judge en CI. +- **NeMo Guardrails full**: configuración Colang con KB embedding + dialog + rails (reintroducir `nemoguardrails` en requirements al integrarlo; hoy el + engine es un stub por keywords configurable vía + `GUARDRAILS_NEMO_ALLOWED_KEYWORDS`). +- **Autenticación**: OAuth2/OIDC con MSAL para entornos Azure AD. Hoy los + endpoints de approve/promote no exigen identidad (el "quién" es declarativo). - **OpenTelemetry**: spans por nodo del grafo, métricas de violaciones por validador. - **Multi-tenant**: `tenant_id` aislado por registry, política y datos. -- **Evaluadores LLM-as-judge**: regression suite que ejecuta cada agente - sobre escenarios canónicos y mide deriva. +- **Evaluadores LLM-as-judge**: complementar la evaluación por escenarios + (ya implementada como gate de promoción) con un juez LLM que mida deriva + de calidad, no solo violaciones de guardrails. - **UI de aprobación con SLA**: cola Kanban, reasignación entre operadores, alertas cuando un HITL rebasa SLA. - **Persistencia migrable a Postgres + pgvector**: requerido para alta disponibilidad multi-instance. - **Esquema de prompts mejorado**: variables tipadas en system_prompt (Jinja-like), valores por entorno. -- **Promoción explícita draft → active** vía PR/aprobación. +- **Deployments**: promoción de agentes validados a endpoints gestionados + (Azure/AWS/GCP/on-prem) conservando políticas y auditoría. + +## Hecho (antes en este roadmap) + +- ~~Promoción explícita draft → active vía aprobación~~ — implementada: + `/api/promotions` + página Promotions con gate de evaluación por versión. +- ~~Fallback LLM~~ — implementado (`LLM_FALLBACK_PROVIDER` + + `FallbackLLMProvider`). diff --git a/docs/product-voice.md b/docs/product-voice.md new file mode 100644 index 0000000..b1a448b --- /dev/null +++ b/docs/product-voice.md @@ -0,0 +1,51 @@ +# Product voice (Forja UI) + +All **user-visible** strings in the web UI, API error messages shown in the UI, and **new code** (identifiers, comments, docstrings, logs) must be in **English**. + +Agent YAML content (`purpose`, `system_prompt`, examples) may use any language required by the domain. + +## Tone + +- Operational and precise, like an internal governance console. +- Short labels; full sentences only where explanation helps. +- No role-play, lore, or game mechanics. + +## Navigation: one page, six stages + +The UI is a single page (`/`) organised as the lifecycle stages. The header nav +contains anchors to each stage; legacy multi-page routes 301-redirect to them. + +| Stage | Anchor | Content | +|-------|--------|---------| +| 01 Define | `#define` | Agents (all versions + states) and guardrail policies | +| 02 Run | `#run` | Demo button + free-form governed execution | +| 03 Approve | `#approve` | HITL queue for paused runs | +| 04 Train & evaluate | `#train` | Training runs, evaluation, promotion request | +| 05 Promote | `#promote` | Governance promotion queue | +| 06 Audit | `#audit` | Completed runs | + +## Execution status labels + +| Internal status | UI label | +|-----------------|----------| +| `running` | Running | +| `awaiting_approval` | Pending approval | +| `completed` | Completed | +| `failed` | Failed | +| `blocked_by_guardrail` | Blocked by policy | + +## Forbidden terms (UI copy) + +Do not use these in templates, buttons, or dynamic HTML from `web/ui.py`: + +- blade, steel, strike, anvil, forge (as verb), temper, tempered, forging +- judgment chamber, bless, shatter, arsenal (as place name) +- hearth, crucible, battle (as station names) +- FORGE MASTER, level, heat, XP, station I/II/III +- Fire • Hammer • Judgment and similar slogans + +The product name **Forja** is allowed in the header and documentation. + +## Development + +Follow [`karpathy.md`](../karpathy.md): think before coding, simplicity first, surgical changes, verifiable success criteria (`make test`, `make lint`). \ No newline at end of file diff --git a/docs/project-progress.html b/docs/project-progress.html new file mode 100644 index 0000000..377c020 --- /dev/null +++ b/docs/project-progress.html @@ -0,0 +1,220 @@ + + + + + + Forja — Project Progress + + + +
+ +

Status snapshot · 2 June 2026

+

Forja

+

Governed control plane for LLM agents and models — registry, guardrails, HITL, training orchestration, and promotion gates.

+ +

Product direction

+

+ Forja is evolving from an agent governance runtime into a full model lifecycle forge: + define agents and policies (Studio), validate with guardrails and human review (Runs, Approvals), + orchestrate external fine-tuning (Azure ML), evaluate on canonical scenarios, and promote draft versions to production (Promotions). +

+
+
StudioAgents, policies, datasets, base models (YAML)
+
ValidationRuns, guardrails, runtime HITL
+
TrainingAzure ML SDK or mock/stub backends
+
GovernancePost-train eval gate + promotion HITL
+
+ +

Completed in this iteration

+ +

Phase 0 — Professional UI (English)

+
    +
  • Removed gamified copy (Hearth, Anvil, blades, levels, etc.)
    • +
  • Navigation: Agents, Runs, Approvals, Promotions, Run history, Deployments
    • +
  • docs/product-voice.md and karpathy.md referenced from README
    • +
  • Terminal runs from the UI persist to data/executions.jsonl
    • +
+ +

Phase A — MLOps orchestration (foundation)

+
    +
  • Versioned datasets/ and models/ registries
    • +
  • TrainingBackend strategy: mock and azure_ml
    • +
  • REST: /api/datasets, /api/models, /api/training/runs
    • +
  • Training runs stored under data/training_runs/
    • +
+ +

Phase B — Eval gate and governance promotion

+
    +
  • Post-train evaluation over agents/<name>/examples/*.txt
    • +
  • POST /api/training/runs/{id}/evaluate — blocks promotion if guardrails fail
    • +
  • /api/promotions — request, approve, reject; activates agent version in registry
    • +
  • HTMX page /promotions for the governance queue
    • +
+ +

Azure ML SDK integration

+
    +
  • azure-ai-ml + azure-identity (DefaultAzureCredential)
    • +
  • Submits command jobs when AZURE_ML_* workspace settings are set
    • +
  • Falls back to stub mode locally when workspace is not configured
    • +
  • Placeholder script: core/src/forja_core/training/_azure_job_bundle/train.py
    • +
+ +

Already in place (MVP core)

+
    +
  • LangGraph runtime with SQLite checkpoints (HITL survives restarts)
    • +
  • Guardrails AI engine + policy YAML
    • +
  • Agent registry with Git-like versioning
    • +
  • REST API under /api and HTMX UI on port 8000
    • +
+ +

How to run the project

+ +

Option 1 — Docker (recommended)

+ 
cp .env.example .env
+docker compose up
+

Open http://localhost:8000. Default LLM_PROVIDER=mock works without API keys.

+ +

Option 2 — Local Python

+ 
cp .env.example .env
+python -m venv venv
+source venv/bin/activate
+pip install -r core/requirements.txt
+pip install pytest pytest-asyncio ruff mypy freezegun
+
+cd core/src
+uvicorn forja_core.main:app --reload --host 0.0.0.0 --port 8000
+

Run from repo root with PYTHONPATH=core/src if needed:

+ 
PYTHONPATH=core/src uvicorn forja_core.main:app --reload --port 8000
+ +

Tests

+ 
source venv/bin/activate
+./venv/bin/pytest tests/unit -q          # unit
+./venv/bin/pytest tests/integration -q   # integration (slower)
+make lint                                 # ruff + mypy (if tools on PATH)
+ +

Quick demo (3 steps)

+
    +
  1. Agents — /agents → inspect incident_analyzer
    2. +
  2. Runs — /run → scenario 01_sip_registration_drop (may trigger Approvals)
    3. +
  3. Approvals / Promotions — /approvals and /promotions
    4. +
+ +

Key API endpoints

+ + + + + + + + + +
AreaEndpoint
HealthGET /health
Invoke agentPOST /api/agents/{name}/invoke
Training runPOST /api/training/runs
Post-train evalPOST /api/training/runs/{id}/evaluate
PromotionPOST /api/promotions → .../approve
+ +

Next steps (when you resume)

+
    +
  • Wire Forja datasets to Azure ML data assets (not only env vars)
    • +
  • Replace placeholder train.py with real fine-tuning script
    • +
  • Multi-tenant, OIDC, Postgres (enterprise roadmap)
    • +
  • LLM-as-judge regression suite in CI
    • +
+ +

Documentation

+ + + +
+ + \ No newline at end of file diff --git a/models/gpt4o_lora_base/index.yaml b/models/gpt4o_lora_base/index.yaml new file mode 100644 index 0000000..b76b92f --- /dev/null +++ b/models/gpt4o_lora_base/index.yaml @@ -0,0 +1,8 @@ +name: gpt4o_lora_base +versions: + - id: v1 + hash: pending + author: Juan + message: LoRA target for incident analyzer fine-tunes. + created_at: 2026-06-01T10:00:00Z +active_version: v1 \ No newline at end of file diff --git a/models/gpt4o_lora_base/versions/v1.yaml b/models/gpt4o_lora_base/versions/v1.yaml new file mode 100644 index 0000000..1c4d7e7 --- /dev/null +++ b/models/gpt4o_lora_base/versions/v1.yaml @@ -0,0 +1,8 @@ +name: gpt4o_lora_base +version: v1 +owner: Juan +purpose: Base model reference for LoRA fine-tuning via external MLOps. +state: active +base_model_id: gpt-4o +adaptation: lora +updated_at: 2026-06-01T10:00:00Z \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d4445e8..b6af361 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,11 +9,8 @@ description = "Plataforma profesional de gobernanza de agentes IA" requires-python = ">=3.11" authors = [{name = "Juan", email = "jm0x@proton.me"}] -[project.dependencies] -httpx = ">=0.27,<0.28" - [tool.setuptools.packages.find] -where = ["core/src", "common/src"] +where = ["core/src"] namespaces = false [tool.ruff] @@ -25,7 +22,7 @@ select = ["E", "F", "W", "I", "N", "UP", "B", "C4", "SIM", "ASYNC", "RUF"] ignore = ["E501"] [tool.ruff.lint.isort] -known-first-party = ["forja_core", "forja_common"] +known-first-party = ["forja_core"] [tool.mypy] python_version = "3.11" @@ -36,14 +33,20 @@ disallow_untyped_defs = true plugins = ["pydantic.mypy"] [[tool.mypy.overrides]] -module = ["guardrails.*", "presidio_analyzer.*", "nemoguardrails.*", "langgraph.*", "yaml", "jsonschema"] +module = [ + "presidio_analyzer.*", + "langgraph.*", + "yaml", + "jsonschema", + "azure.*", +] ignore_missing_imports = true [tool.pytest.ini_options] asyncio_mode = "auto" testpaths = ["tests"] addopts = "-ra -q --strict-markers" -pythonpath = ["core/src", "common/src"] +pythonpath = ["core/src"] markers = [ "integration: integration tests (slower, may touch FS)", ] diff --git a/resume.txt b/resume.txt new file mode 100644 index 0000000..d73ff04 --- /dev/null +++ b/resume.txt @@ -0,0 +1 @@ +claude --resume 0bd8c3c4-89d9-4d22-958a-36ef2b8b366c diff --git a/tests/fixtures/agents/test_agent/examples/basic.txt b/tests/fixtures/agents/test_agent/examples/basic.txt new file mode 100644 index 0000000..a4af1ad --- /dev/null +++ b/tests/fixtures/agents/test_agent/examples/basic.txt @@ -0,0 +1 @@ +Routine platform health check. No user impact reported. \ No newline at end of file diff --git a/tests/fixtures/agents/test_agent/index.yaml b/tests/fixtures/agents/test_agent/index.yaml index 013e41e..9c85036 100644 --- a/tests/fixtures/agents/test_agent/index.yaml +++ b/tests/fixtures/agents/test_agent/index.yaml @@ -3,6 +3,11 @@ versions: - id: v1 hash: feedface author: pytest - message: fixture inicial + message: fixture initial created_at: 2026-01-01T00:00:00Z -active_version: v1 + - id: v2 + hash: draftv2 + author: pytest + message: draft for promotion tests + created_at: 2026-06-01T00:00:00Z +active_version: v1 \ No newline at end of file diff --git a/tests/fixtures/agents/test_agent/versions/v2.yaml b/tests/fixtures/agents/test_agent/versions/v2.yaml new file mode 100644 index 0000000..00ed88e --- /dev/null +++ b/tests/fixtures/agents/test_agent/versions/v2.yaml @@ -0,0 +1,18 @@ +name: test_agent +version: v2 +owner: pytest +purpose: Draft version for governance promotion tests. +state: draft +guardrails: [test_policy] +llm: + provider: mock + model: gpt-4o + temperature: 0.2 + max_tokens: 2000 +system_prompt: | + You are a test agent. Return JSON with severity and proposed_actions. +output_schema: + type: object + required: [severity] +risk_threshold_for_hitl: 4 +updated_at: 2026-06-01T00:00:00Z \ No newline at end of file diff --git a/tests/fixtures/datasets/incident_sft/index.yaml b/tests/fixtures/datasets/incident_sft/index.yaml new file mode 100644 index 0000000..5d647a6 --- /dev/null +++ b/tests/fixtures/datasets/incident_sft/index.yaml @@ -0,0 +1,8 @@ +name: incident_sft +versions: + - id: v1 + hash: pending + author: test + message: fixture + created_at: 2026-01-01T00:00:00Z +active_version: v1 \ No newline at end of file diff --git a/tests/fixtures/datasets/incident_sft/versions/v1.yaml b/tests/fixtures/datasets/incident_sft/versions/v1.yaml new file mode 100644 index 0000000..652b844 --- /dev/null +++ b/tests/fixtures/datasets/incident_sft/versions/v1.yaml @@ -0,0 +1,8 @@ +name: incident_sft +version: v1 +owner: test +purpose: Test dataset +state: active +format: jsonl +records_path: records/v1.jsonl +updated_at: 2026-01-01T00:00:00Z \ No newline at end of file diff --git a/tests/fixtures/models/gpt4o_lora_base/index.yaml b/tests/fixtures/models/gpt4o_lora_base/index.yaml new file mode 100644 index 0000000..42fd225 --- /dev/null +++ b/tests/fixtures/models/gpt4o_lora_base/index.yaml @@ -0,0 +1,8 @@ +name: gpt4o_lora_base +versions: + - id: v1 + hash: pending + author: test + message: fixture + created_at: 2026-01-01T00:00:00Z +active_version: v1 \ No newline at end of file diff --git a/tests/fixtures/models/gpt4o_lora_base/versions/v1.yaml b/tests/fixtures/models/gpt4o_lora_base/versions/v1.yaml new file mode 100644 index 0000000..d225f0d --- /dev/null +++ b/tests/fixtures/models/gpt4o_lora_base/versions/v1.yaml @@ -0,0 +1,8 @@ +name: gpt4o_lora_base +version: v1 +owner: test +purpose: Test model +state: active +base_model_id: gpt-4o +adaptation: lora +updated_at: 2026-01-01T00:00:00Z \ No newline at end of file diff --git a/tests/integration/test_invoke_happy_path.py b/tests/integration/test_invoke_happy_path.py index ca23382..f6ba140 100644 --- a/tests/integration/test_invoke_happy_path.py +++ b/tests/integration/test_invoke_happy_path.py @@ -4,9 +4,7 @@ from pathlib import Path import pytest -EXAMPLES = ( - Path(__file__).parent.parent.parent / "agents" / "incident_analyzer" / "examples" -) +EXAMPLES = Path(__file__).parent.parent.parent / "agents" / "incident_analyzer" / "examples" @pytest.mark.integration diff --git a/tests/integration/test_invoke_hitl.py b/tests/integration/test_invoke_hitl.py index 3e71739..4b5871d 100644 --- a/tests/integration/test_invoke_hitl.py +++ b/tests/integration/test_invoke_hitl.py @@ -4,9 +4,7 @@ from pathlib import Path import pytest -EXAMPLES = ( - Path(__file__).parent.parent.parent / "agents" / "incident_analyzer" / "examples" -) +EXAMPLES = Path(__file__).parent.parent.parent / "agents" / "incident_analyzer" / "examples" @pytest.mark.integration diff --git a/tests/integration/test_invoke_resume_after_restart.py b/tests/integration/test_invoke_resume_after_restart.py index c66a178..dd4a5eb 100644 --- a/tests/integration/test_invoke_resume_after_restart.py +++ b/tests/integration/test_invoke_resume_after_restart.py @@ -10,9 +10,7 @@ from pathlib import Path import pytest -EXAMPLES = ( - Path(__file__).parent.parent.parent / "agents" / "incident_analyzer" / "examples" -) +EXAMPLES = Path(__file__).parent.parent.parent / "agents" / "incident_analyzer" / "examples" @pytest.mark.integration diff --git a/tests/unit/test_api_executions.py b/tests/unit/test_api_executions.py index 7f73fb3..7db9544 100644 --- a/tests/unit/test_api_executions.py +++ b/tests/unit/test_api_executions.py @@ -34,7 +34,9 @@ def client() -> TestClient: def test_invoke_devuelve_execution_completada(client: TestClient) -> None: - r = client.post("/api/agents/incident_analyzer/invoke", json={"input": "degradación MOS pool SBC"}) + r = client.post( + "/api/agents/incident_analyzer/invoke", json={"input": "degradación MOS pool SBC"} + ) assert r.status_code == 200 body = r.json() assert body["status"] == "completed" diff --git a/tests/unit/test_api_policies_violations.py b/tests/unit/test_api_policies_violations.py index 7c8d2db..4476653 100644 --- a/tests/unit/test_api_policies_violations.py +++ b/tests/unit/test_api_policies_violations.py @@ -59,7 +59,9 @@ def test_violations_vacio_si_no_hay(client: TestClient) -> None: assert r.json() == [] -def _violation(trace_id: object, validator: str, severity: str, blocked: bool) -> GuardrailViolation: +def _violation( + trace_id: object, validator: str, severity: str, blocked: bool +) -> GuardrailViolation: return GuardrailViolation( trace_id=trace_id, # type: ignore[arg-type] timestamp=datetime.now(UTC), diff --git a/tests/unit/test_api_promotions.py b/tests/unit/test_api_promotions.py new file mode 100644 index 0000000..8782f71 --- /dev/null +++ b/tests/unit/test_api_promotions.py @@ -0,0 +1,173 @@ +"""Governance promotion flow: train → evaluate → request → approve.""" + +import shutil +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from forja_core.api import deps +from forja_core.main import create_app + +FIXTURES = Path(__file__).parent.parent / "fixtures" + + +@pytest.fixture(autouse=True) +def patch_settings(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + agents_dir = tmp_path / "agents" + shutil.copytree(FIXTURES / "agents", agents_dir) + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + monkeypatch.setenv("AGENTS_DIR", str(agents_dir)) + monkeypatch.setenv("POLICIES_DIR", str(FIXTURES / "policies")) + monkeypatch.setenv("DATASETS_DIR", str(FIXTURES / "datasets")) + monkeypatch.setenv("MODELS_DIR", str(FIXTURES / "models")) + monkeypatch.setenv("TRAINING_BACKEND", "mock") + for fn in ( + deps.get_settings, + deps.get_registry, + deps.get_policy_store, + deps.get_dataset_store, + deps.get_model_store, + deps.get_training_backend, + deps.get_orchestrator, + deps.get_llm_provider, + deps.get_guardrail_engine, + ): + fn.cache_clear() + + +def test_promotion_flow_approve_activates_draft_version() -> None: + client = TestClient(create_app()) + + run = client.post( + "/api/training/runs", + json={ + "dataset_name": "incident_sft", + "model_name": "gpt4o_lora_base", + "agent_name": "test_agent", + "agent_version": "v2", + }, + ).json() + run_id = run["id"] + + # Evaluation must cover the candidate version, not the active one. + eval_report = client.post(f"/api/training/runs/{run_id}/evaluate").json() + assert eval_report["passed"] is True + assert eval_report["agent_version"] == "v2" + + promo = client.post( + "/api/promotions", + json={ + "agent_name": "test_agent", + "agent_version": "v2", + "training_run_id": run_id, + "evaluation_report_id": eval_report["id"], + }, + ) + assert promo.status_code == 201 + request_id = promo.json()["id"] + + approve = client.post( + f"/api/promotions/{request_id}/approve", + json={"approved_by": "ops-lead", "comment": "Eval gate passed"}, + ) + assert approve.status_code == 200 + + agent = client.get("/api/agents/test_agent").json() + assert agent["version"] == "v2" + assert agent["state"] == "active" + + +def test_promotion_blocked_when_eval_failed(tmp_path: Path) -> None: + """Promotion returns 422 if linked evaluation did not pass.""" + from datetime import UTC, datetime + from uuid import uuid4 + + from forja_core.api.evaluation_persistence import save_evaluation + from forja_core.api.training_persistence import save_training_run + from forja_core.domain.training import ( + EvaluationReport, + TrainingHyperparameters, + TrainingJobSpec, + TrainingRun, + ) + + client = TestClient(create_app()) + settings = deps.get_settings() + run_id = uuid4() + now = datetime.now(UTC) + spec = TrainingJobSpec( + dataset_name="incident_sft", + dataset_version="v1", + model_name="gpt4o_lora_base", + model_version="v1", + agent_name="test_agent", + hyperparameters=TrainingHyperparameters(), + ) + save_training_run( + settings.data_dir, + TrainingRun( + id=run_id, + backend="mock", + status="succeeded", + spec=spec, + external_job_id="mock:x", + created_at=now, + updated_at=now, + ), + ) + report = EvaluationReport( + id=uuid4(), + training_run_id=run_id, + agent_name="test_agent", + agent_version="v1", + passed=False, + scenario_count=1, + violation_count=1, + created_at=now, + notes="forced failure", + ) + save_evaluation(settings.data_dir, report) + + r = client.post( + "/api/promotions", + json={ + "agent_name": "test_agent", + "agent_version": "v2", + "training_run_id": str(run_id), + "evaluation_report_id": str(report.id), + }, + ) + assert r.status_code == 422 + + +def test_promotion_blocked_when_eval_covers_other_version() -> None: + """Evaluating the active version must not unlock promoting a different draft.""" + client = TestClient(create_app()) + + # No agent_version on the run → evaluation covers the active version (v1). + run = client.post( + "/api/training/runs", + json={ + "dataset_name": "incident_sft", + "model_name": "gpt4o_lora_base", + "agent_name": "test_agent", + }, + ).json() + run_id = run["id"] + + eval_report = client.post(f"/api/training/runs/{run_id}/evaluate").json() + assert eval_report["passed"] is True + assert eval_report["agent_version"] == "v1" + + r = client.post( + "/api/promotions", + json={ + "agent_name": "test_agent", + "agent_version": "v2", + "training_run_id": run_id, + "evaluation_report_id": eval_report["id"], + }, + ) + assert r.status_code == 422 + assert "evaluate the candidate version" in r.json()["detail"] diff --git a/tests/unit/test_api_training.py b/tests/unit/test_api_training.py new file mode 100644 index 0000000..a97a943 --- /dev/null +++ b/tests/unit/test_api_training.py @@ -0,0 +1,70 @@ +"""Tests for /api/training/runs and registry endpoints.""" + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from forja_core.api import deps +from forja_core.main import create_app + +FIXTURES = Path(__file__).parent.parent / "fixtures" + + +@pytest.fixture(autouse=True) +def patch_settings(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + monkeypatch.setenv("AGENTS_DIR", str(FIXTURES / "agents")) + monkeypatch.setenv("POLICIES_DIR", str(FIXTURES / "policies")) + monkeypatch.setenv("DATASETS_DIR", str(FIXTURES / "datasets")) + monkeypatch.setenv("MODELS_DIR", str(FIXTURES / "models")) + monkeypatch.setenv("TRAINING_BACKEND", "mock") + for fn in ( + deps.get_settings, + deps.get_registry, + deps.get_policy_store, + deps.get_dataset_store, + deps.get_model_store, + deps.get_training_backend, + deps.get_orchestrator, + deps.get_llm_provider, + deps.get_guardrail_engine, + ): + fn.cache_clear() + + +def test_list_datasets() -> None: + r = TestClient(create_app()).get("/api/datasets") + assert r.status_code == 200 + assert any(d["name"] == "incident_sft" for d in r.json()) + + +def test_create_training_run_mock_backend(tmp_path: Path) -> None: + client = TestClient(create_app()) + r = client.post( + "/api/training/runs", + json={ + "dataset_name": "incident_sft", + "model_name": "gpt4o_lora_base", + }, + ) + assert r.status_code == 201 + body = r.json() + assert body["status"] == "succeeded" + assert body["backend"] == "mock" + assert body["external_job_id"].startswith("mock:") + assert len(body["artifact_refs"]) >= 1 + run_id = body["id"] + assert (tmp_path / "training_runs" / f"{run_id}.json").exists() + + r2 = client.get(f"/api/training/runs/{run_id}") + assert r2.status_code == 200 + assert r2.json()["id"] == run_id + + +def test_create_training_run_unknown_dataset_404() -> None: + r = TestClient(create_app()).post( + "/api/training/runs", + json={"dataset_name": "missing", "model_name": "gpt4o_lora_base"}, + ) + assert r.status_code == 404 diff --git a/tests/unit/test_azure_ml_sdk.py b/tests/unit/test_azure_ml_sdk.py new file mode 100644 index 0000000..16edb82 --- /dev/null +++ b/tests/unit/test_azure_ml_sdk.py @@ -0,0 +1,96 @@ +"""Azure ML SDK backend with mocked MLClient.""" + +from unittest.mock import MagicMock, patch +from uuid import uuid4 + +import pytest + +from forja_core.config import Settings +from forja_core.domain.training import TrainingHyperparameters, TrainingJobSpec +from forja_core.training.azure_ml import AzureMLTrainingBackend +from forja_core.training.azure_ml_sdk import poll_job_status, submit_command_job + + +@pytest.fixture +def azure_settings() -> Settings: + return Settings( + training_backend="azure_ml", + azure_ml_subscription_id="sub-123", + azure_ml_resource_group="rg-forja", + azure_ml_workspace_name="ws-forja", + azure_ml_compute="cpu-cluster", + ) + + +@pytest.fixture +def spec() -> TrainingJobSpec: + return TrainingJobSpec( + dataset_name="incident_sft", + dataset_version="v1", + model_name="gpt4o_lora_base", + model_version="v1", + agent_name="test_agent", + hyperparameters=TrainingHyperparameters(), + ) + + +@patch("forja_core.training.azure_ml_sdk._ml_client") +def test_submit_command_job_returns_azureml_prefix( + mock_client_factory: MagicMock, + azure_settings: Settings, + spec: TrainingJobSpec, +) -> None: + client = MagicMock() + mock_client_factory.return_value = client + created = MagicMock() + created.name = "forja-job-abc" + client.jobs.create_or_update.return_value = created + + run_id = uuid4() + external_id = submit_command_job(azure_settings, spec, run_id) + + assert external_id == "azureml:forja-job-abc" + client.jobs.create_or_update.assert_called_once() + + +@patch("forja_core.training.azure_ml_sdk._ml_client") +def test_poll_job_status_maps_completed( + mock_client_factory: MagicMock, + azure_settings: Settings, +) -> None: + client = MagicMock() + mock_client_factory.return_value = client + job = MagicMock() + job.status = "Completed" + client.jobs.get.return_value = job + + result = poll_job_status(azure_settings, "forja-job-abc") + + assert result.status == "succeeded" + assert len(result.artifacts) == 1 + assert "azureml://" in result.artifacts[0].uri + + +@pytest.mark.asyncio +async def test_backend_uses_sdk_when_configured( + azure_settings: Settings, + spec: TrainingJobSpec, +) -> None: + backend = AzureMLTrainingBackend(azure_settings) + assert backend._use_sdk is True + + with patch( + "forja_core.training.azure_ml_sdk.submit_command_job", + return_value="azureml:job-1", + ) as mock_submit: + job_id = await backend.submit(spec, uuid4()) + assert job_id == "azureml:job-1" + mock_submit.assert_called_once() + + with patch( + "forja_core.training.azure_ml_sdk.poll_job_status", + return_value=MagicMock(status="running", message=None, artifacts=[]), + ) as mock_poll: + status = await backend.status("azureml:job-1") + assert status.status == "running" + mock_poll.assert_called_once_with(azure_settings, "job-1") diff --git a/tests/unit/test_llm_factory.py b/tests/unit/test_llm_factory.py index 33c6a49..bdae0dd 100644 --- a/tests/unit/test_llm_factory.py +++ b/tests/unit/test_llm_factory.py @@ -1,9 +1,13 @@ """Tests del factory LLM.""" +from typing import Any + import pytest from forja_core.config import Settings +from forja_core.llm.base import CompletionResult, Message from forja_core.llm.factory import build_llm_provider +from forja_core.llm.fallback import FallbackLLMProvider from forja_core.llm.mock import MockProvider @@ -18,3 +22,46 @@ def test_factory_azure_requiere_credenciales() -> None: s = Settings(llm_provider="azure", _env_file=None) # type: ignore[call-arg] with pytest.raises(ValueError): build_llm_provider(s) + + +def test_factory_envuelve_con_fallback_cuando_difiere() -> None: + s = Settings( + llm_provider="openai", + llm_fallback_provider="mock", + openai_api_key="sk-test", + _env_file=None, + ) # type: ignore[call-arg] + p = build_llm_provider(s) + assert isinstance(p, FallbackLLMProvider) + assert p.name == "openai+fallback:mock" + + +def test_factory_ignora_fallback_igual_al_primario() -> None: + s = Settings(llm_provider="mock", llm_fallback_provider="mock", _env_file=None) # type: ignore[call-arg] + p = build_llm_provider(s) + assert isinstance(p, MockProvider) + + +class _FailingProvider: + name = "failing" + + async def complete( + self, + messages: list[Message], + schema: dict[str, Any] | None = None, + temperature: float = 0.2, + max_tokens: int = 2000, + ) -> CompletionResult: + raise RuntimeError("primary down") + + +async def test_fallback_delega_cuando_el_primario_falla() -> None: + provider = FallbackLLMProvider(_FailingProvider(), MockProvider()) + result = await provider.complete([Message(role="user", content="mos degradation")]) + assert result.model.startswith("mock") + + +async def test_fallback_no_interviene_si_el_primario_responde() -> None: + provider = FallbackLLMProvider(MockProvider(), _FailingProvider()) + result = await provider.complete([Message(role="user", content="mos degradation")]) + assert result.model.startswith("mock") diff --git a/tests/unit/test_ml_registry.py b/tests/unit/test_ml_registry.py new file mode 100644 index 0000000..23fdf4a --- /dev/null +++ b/tests/unit/test_ml_registry.py @@ -0,0 +1,23 @@ +"""Tests for dataset and model YAML registries.""" + +from pathlib import Path + +from forja_core.registry.dataset_store import FileSystemDatasetStore +from forja_core.registry.model_store import FileSystemModelStore + +FIXTURES = Path(__file__).parent.parent / "fixtures" + + +def test_get_dataset_active_version() -> None: + store = FileSystemDatasetStore(FIXTURES / "datasets") + ds = store.get_dataset("incident_sft") + assert ds.name == "incident_sft" + assert ds.version == "v1" + assert ds.format == "jsonl" + + +def test_get_model_active_version() -> None: + store = FileSystemModelStore(FIXTURES / "models") + m = store.get_model("gpt4o_lora_base") + assert m.adaptation == "lora" + assert m.base_model_id == "gpt-4o" diff --git a/tests/unit/test_post_train_eval.py b/tests/unit/test_post_train_eval.py new file mode 100644 index 0000000..5a807b8 --- /dev/null +++ b/tests/unit/test_post_train_eval.py @@ -0,0 +1,63 @@ +"""Post-training evaluation over canonical scenarios.""" + +from pathlib import Path + +import pytest + +from forja_core.api.deps import ( + get_guardrail_engine, + get_llm_provider, + get_orchestrator, + get_policy_store, + get_registry, + get_settings, +) +from forja_core.evaluation.post_train import run_post_train_evaluation +from forja_core.evaluation.scenarios import load_agent_scenarios + +FIXTURES = Path(__file__).parent.parent / "fixtures" + + +@pytest.fixture(autouse=True) +def patch_settings(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + monkeypatch.setenv("AGENTS_DIR", str(FIXTURES / "agents")) + monkeypatch.setenv("POLICIES_DIR", str(FIXTURES / "policies")) + for fn in ( + get_settings, + get_registry, + get_policy_store, + get_orchestrator, + get_llm_provider, + get_guardrail_engine, + ): + fn.cache_clear() + + +@pytest.mark.asyncio +async def test_load_scenarios_from_examples() -> None: + scenarios = load_agent_scenarios(FIXTURES / "agents", "test_agent") + assert len(scenarios) >= 1 + assert scenarios[0][0].endswith(".txt") + + +@pytest.mark.asyncio +async def test_post_train_eval_passes_for_test_agent() -> None: + registry = get_registry() + policies = get_policy_store() + orchestrator = get_orchestrator() + settings = get_settings() + agent = registry.get_agent("test_agent") + policy = policies.get_policy(agent.guardrails[0]) + from uuid import uuid4 + + report = await run_post_train_evaluation( + training_run_id=uuid4(), + agent_def=agent, + policy=policy, + orchestrator=orchestrator, + agents_dir=settings.agents_dir, + ) + assert report.scenario_count >= 1 + assert report.passed is True + assert report.violation_count == 0 diff --git a/tests/unit/test_registry.py b/tests/unit/test_registry.py index 34a5523..f2bd38f 100644 --- a/tests/unit/test_registry.py +++ b/tests/unit/test_registry.py @@ -20,8 +20,8 @@ def test_get_agent_carga_version_activa() -> None: def test_list_versions_devuelve_metadata() -> None: r = FileSystemAgentRegistry(FIXTURES) metas = r.list_versions("test_agent") - assert len(metas) == 1 - assert metas[0].id == "v1" + assert len(metas) == 2 + assert {m.id for m in metas} == {"v1", "v2"} def test_upsert_crea_nueva_version_y_actualiza_index(tmp_path: Path) -> None: diff --git a/tests/unit/test_runtime_graph.py b/tests/unit/test_runtime_graph.py index c7d43b1..5e4d3e1 100644 --- a/tests/unit/test_runtime_graph.py +++ b/tests/unit/test_runtime_graph.py @@ -106,7 +106,10 @@ async def test_grafo_bloquea_por_guardrail_de_entrada(tmp_path: Path) -> None: version="v1", description="bloquea emails en la entrada", input_validators=[ - {"type": "detect_pii", "config": {"entities": ["EMAIL_ADDRESS"], "severity_on_match": "block"}} + { + "type": "detect_pii", + "config": {"entities": ["EMAIL_ADDRESS"], "severity_on_match": "block"}, + } ], output_validators=[], ) diff --git a/tests/unit/test_runtime_orchestrator.py b/tests/unit/test_runtime_orchestrator.py index a1bf5d0..85b358e 100644 --- a/tests/unit/test_runtime_orchestrator.py +++ b/tests/unit/test_runtime_orchestrator.py @@ -79,7 +79,9 @@ async def test_snapshot_relee_estado_sin_avanzar(orchestrator: AgentOrchestrator async def test_snapshot_trace_id_desconocido_devuelve_none(orchestrator: AgentOrchestrator) -> None: - assert await orchestrator.snapshot(agent_def=_agent(), policy=_policy(), trace_id=uuid4()) is None + assert ( + await orchestrator.snapshot(agent_def=_agent(), policy=_policy(), trace_id=uuid4()) is None + ) async def test_invoke_pausa_hitl_y_resume_aprueba(orchestrator: AgentOrchestrator) -> None: @@ -137,16 +139,12 @@ async def test_invoke_bloqueado_por_guardrail(orchestrator: AgentOrchestrator) - async def test_estado_persiste_entre_instancias(tmp_path: Path) -> None: agent = _agent() - o1 = AgentOrchestrator( - provider=MockProvider(), engine=GuardrailsAIEngine(), data_dir=tmp_path - ) + o1 = AgentOrchestrator(provider=MockProvider(), engine=GuardrailsAIEngine(), data_dir=tmp_path) paused = await o1.invoke(agent_def=agent, policy=_policy(), user_input="caída registros sip") assert paused.status == "awaiting_approval" # Nueva instancia (simula reinicio del proceso) apuntando al mismo data_dir. - o2 = AgentOrchestrator( - provider=MockProvider(), engine=GuardrailsAIEngine(), data_dir=tmp_path - ) + o2 = AgentOrchestrator(provider=MockProvider(), engine=GuardrailsAIEngine(), data_dir=tmp_path) resumed = await o2.resume( agent_def=agent, policy=_policy(), diff --git a/tests/unit/test_training_backend.py b/tests/unit/test_training_backend.py new file mode 100644 index 0000000..7a4bfbb --- /dev/null +++ b/tests/unit/test_training_backend.py @@ -0,0 +1,43 @@ +"""Tests for mock and Azure ML stub training backends.""" + +from uuid import uuid4 + +import pytest + +from forja_core.config import Settings +from forja_core.domain.training import TrainingHyperparameters, TrainingJobSpec +from forja_core.training.azure_ml import AzureMLTrainingBackend +from forja_core.training.mock import MockTrainingBackend + + +@pytest.fixture +def spec() -> TrainingJobSpec: + return TrainingJobSpec( + dataset_name="incident_sft", + dataset_version="v1", + model_name="gpt4o_lora_base", + model_version="v1", + hyperparameters=TrainingHyperparameters(epochs=2), + ) + + +@pytest.mark.asyncio +async def test_mock_submit_and_status_succeeds(spec: TrainingJobSpec) -> None: + backend = MockTrainingBackend() + run_id = uuid4() + job_id = await backend.submit(spec, run_id) + assert job_id == f"mock:{run_id}" + status = await backend.status(job_id) + assert status.status == "succeeded" + assert len(status.artifacts) == 1 + + +@pytest.mark.asyncio +async def test_azure_stub_submit_and_status_succeeds(spec: TrainingJobSpec) -> None: + backend = AzureMLTrainingBackend(settings=Settings()) + run_id = uuid4() + job_id = await backend.submit(spec, run_id) + assert job_id.startswith("azureml-stub:") + status = await backend.status(job_id) + assert status.status == "succeeded" + assert "azureml://" in status.artifacts[0].uri diff --git a/tests/unit/test_ui_pages.py b/tests/unit/test_ui_pages.py new file mode 100644 index 0000000..0e5d8c3 --- /dev/null +++ b/tests/unit/test_ui_pages.py @@ -0,0 +1,185 @@ +"""Single-page UI tests: stages render, fragments refresh, and the full +click-through lifecycle (run → approve → train → evaluate → promote) works.""" + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from forja_core.api import deps +from forja_core.main import create_app + +FIXTURES = Path(__file__).parent.parent / "fixtures" + + +@pytest.fixture(autouse=True) +def patch_settings(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + import shutil + + # Agents are copied to tmp because the promotion test mutates the registry. + agents_dir = tmp_path / "agents" + shutil.copytree(FIXTURES / "agents", agents_dir) + monkeypatch.setenv("DATA_DIR", str(tmp_path / "data")) + monkeypatch.setenv("AGENTS_DIR", str(agents_dir)) + monkeypatch.setenv("POLICIES_DIR", str(FIXTURES / "policies")) + monkeypatch.setenv("DATASETS_DIR", str(FIXTURES / "datasets")) + monkeypatch.setenv("MODELS_DIR", str(FIXTURES / "models")) + monkeypatch.setenv("TRAINING_BACKEND", "mock") + monkeypatch.setenv("LLM_PROVIDER", "mock") + for fn in ( + deps.get_settings, + deps.get_registry, + deps.get_policy_store, + deps.get_dataset_store, + deps.get_model_store, + deps.get_training_backend, + deps.get_orchestrator, + deps.get_llm_provider, + deps.get_guardrail_engine, + ): + fn.cache_clear() + + +def test_index_renders_all_six_stages() -> None: + r = TestClient(create_app()).get("/") + assert r.status_code == 200 + for anchor in ( + 'id="define"', + 'id="run"', + 'id="approve"', + 'id="train"', + 'id="promote"', + 'id="audit"', + ): + assert anchor in r.text + # Define stage: agent with version/state chips and the policy validators. + assert "test_agent" in r.text + assert "v1 · active" in r.text + assert "v2 · draft" in r.text + assert "detect_pii" in r.text + # Run stage: the one-click demo button. + assert "Run demo incident" in r.text + + +def test_run_invoke_completes_and_lands_in_history_fragment() -> None: + client = TestClient(create_app()) + # "mos" maps to a low-risk canonical mock response → completes without HITL. + r = client.post("/run", data={"agent_name": "test_agent", "input": "mos degradation in pool"}) + assert r.status_code == 200 + assert "Completed" in r.text + assert "validate_input" in r.text + assert "finalize" in r.text + + h = client.get("/fragments/history") + assert h.status_code == 200 + assert "test_agent" in h.text + + +def test_run_invoke_hitl_appears_in_approvals_fragment() -> None: + client = TestClient(create_app()) + # "hss" maps to a risk-5 action requiring approval → execution pauses in HITL. + r = client.post("/run", data={"agent_name": "test_agent", "input": "hss capacity alarm"}) + assert r.status_code == 200 + assert "Pending approval" in r.text + + a = client.get("/fragments/approvals") + assert a.status_code == 200 + assert "test_agent" in a.text + assert "isolate_replication_link" in a.text + assert "Approve all" in a.text + + +def test_run_invoke_unknown_agent_returns_error() -> None: + client = TestClient(create_app()) + r = client.post("/run", data={"agent_name": "missing", "input": "x"}) + assert r.status_code == 400 + assert "Execution error" in r.text + + +@pytest.mark.parametrize( + ("path", "target"), + [ + ("/agents", "/#define"), + ("/policies", "/#define"), + ("/run", "/#run"), + ("/approvals", "/#approve"), + ("/training", "/#train"), + ("/promotions", "/#promote"), + ("/history", "/#audit"), + ("/armory", "/#audit"), + ("/agents/test_agent", "/#define"), + ], +) +def test_legacy_routes_redirect_to_anchors(path: str, target: str) -> None: + client = TestClient(create_app()) + r = client.get(path, follow_redirects=False) + assert r.status_code == 301 + assert r.headers["location"] == target + + +def test_full_lifecycle_click_through() -> None: + """The exact sequence of requests the page buttons fire, end to end.""" + client = TestClient(create_app()) + + # Stage 02 click: run the demo incident → pauses at the approval gate. + r = client.post("/run", data={"agent_name": "test_agent", "input": "hss capacity alarm"}) + assert "Pending approval" in r.text + + # Stage 03 click: approve all pending actions → run completes. + pending = client.get("/api/executions").json() + trace_id = next(e["trace_id"] for e in pending if e["status"] == "awaiting_approval") + approved = client.post( + f"/api/executions/{trace_id}/approve", + json={"approved_action_ids": [], "comment": "Approved from UI"}, + ).json() + assert approved["status"] == "completed" + assert client.get("/fragments/approvals").text.count("Pending approval") == 0 + assert "test_agent" in client.get("/fragments/history").text + + # Stage 04 click: submit a training run for the draft candidate (v2). + run = client.post( + "/api/training/runs", + json={ + "dataset_name": "incident_sft", + "model_name": "gpt4o_lora_base", + "agent_name": "test_agent", + "agent_version": "v2", + }, + ).json() + assert run["status"] == "succeeded" + frag = client.get("/fragments/training").text + assert "Run evaluation" in frag + + # Stage 04 click: evaluate the candidate. + report = client.post(f"/api/training/runs/{run['id']}/evaluate").json() + assert report["passed"] is True + frag = client.get("/fragments/training").text + assert "Request promotion" in frag + + # Stage 04 click: request promotion → shows as pending in stages 04 and 05. + promo = client.post( + "/api/promotions", + json={ + "agent_name": "test_agent", + "agent_version": "v2", + "training_run_id": run["id"], + "evaluation_report_id": report["id"], + "requested_by": "web-ui", + }, + ) + assert promo.status_code == 201 + assert "Promotion pending" in client.get("/fragments/training").text + assert "test_agent" in client.get("/fragments/promotions").text + + # Stage 05 click: approve the promotion → candidate flips to active. + approve = client.post( + f"/api/promotions/{promo.json()['id']}/approve", + json={"approved_by": "web-ui", "comment": "ok"}, + ) + assert approve.status_code == 200 + assert "✓ Promoted" in client.get("/fragments/training").text + assert "v2 · active" in client.get("/fragments/agents").text + + agent = client.get("/api/agents/test_agent").json() + assert agent["version"] == "v2" + assert agent["state"] == "active"