feat: working TOM agent

This commit is contained in:
2026-10-21 19:47:12 +01:00
commit efe543e6e2
83 changed files with 11779 additions and 0 deletions

0
eval/__init__.py Normal file
View File

32
eval/datasets/cases.json Normal file
View File

@@ -0,0 +1,32 @@
[
{
"id": "case-01-landing-zone-shared",
"what": "A written operating spec, complete enough to run with nobody to ask.",
"spec": "specs/landing-zone-shared.yaml",
"max_revisions": 2,
"max_minutes": 40
},
{
"id": "case-02-from-architecture",
"what": "The solution architecture agent's landing-zone Execution, digested, with who operates it stated and the business outcomes left to the agent.",
"sources": "../../tests/fixtures/architecture/case-08-landing-zone",
"overlay": {
"sourcing": {
"model": "provider_operated",
"delivery_countries": [
"GB",
"PL",
"IN"
]
},
"run": {
"pre_approved": true
},
"intent": {
"business_outcomes": "decide"
}
},
"max_revisions": 3,
"max_minutes": 60
}
]

View File

@@ -0,0 +1,53 @@
# A fictional benchmark: a small landing zone run as a shared service. No customer's
# data is in it. Complete enough to run with nobody to ask.
metadata:
customer_case: bench-landing-zone-shared
engagement:
kind: platform_foundation
intent:
statement: |
We want our landing zone run as a shared service from go-live. A provider should
respond to faults at every hour, and our own platform team keeps the final say
over changes to the network.
business_outcomes:
- {id: BO-1, outcome: New payment products reach customers in weeks rather than months., target: "A change reaches production within a day of approval."}
- {id: BO-2, outcome: Payments are never unavailable for long., target: "Restored within 4 hours."}
- {id: BO-3, outcome: We keep the platform engineers we hire.}
requirements:
- {id: REQ-01, statement: Our platform team approves every change to the network.}
- {id: REQ-02, statement: We receive a monthly report of service levels and cost.}
solution:
cloud_provider: gcp
items:
- {id: S-1, name: Hub network and interconnect, kind: network, product: Cloud Interconnect}
- {id: S-2, name: Audit log store, kind: storage, product: Cloud Storage}
- {id: S-3, name: Deployment pipeline, kind: pipeline, product: Cloud Build}
- {id: S-4, name: Encryption keys, kind: keys, product: Cloud KMS}
conditions:
- {id: K-1, "on": [S-1], condition: The platform is available at least 99.9% of each month.}
- {id: K-2, "on": [S-2], condition: Audit logs cannot be deleted for seven years.}
- {id: K-3, "on": [S-4], condition: Encryption keys are rotated every 90 days.}
build:
duration_days: 36
operations:
support_hours: 24x7
availability_hours: 24x7
change_process: decide
sourcing:
model: shared
delivery_countries: [GB, PL, IN]
organisation:
existing_teams:
- {name: Platform team, party: customer, hours: business, country: GB}
service_management:
tooling: decide
service_levels:
priority_1_response_minutes: 15
recovery_time_hours: 4
compliance:
frameworks: [DORA]
commercial:
currency: GBP
monthly_run_budget: 60000
run:
pre_approved: true

View File

@@ -0,0 +1,208 @@
"""Evaluate the target operating model agent on benchmark cases with real models.
Each case runs one pre-approved Execution through the full assembly (the ADK runtime
and the agent's own Skills and role files), then scores it with eval/scoring.py.
It costs model usage: run it deliberately.
Usage:
GOOGLE_GENAI_USE_ENTERPRISE=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \\
.venv/bin/python -m eval.operating_model_eval [case-id ...]
The model settings are read from the environment or from `.env`. With no case named,
every case in eval/datasets/cases.json runs, one after another. Results are written
under eval/results/<time>/: each Execution's files, and results.json. The exit code
is non-zero when a case fails a check or runs over its time.
"""
from __future__ import annotations
import asyncio
import json
import logging
import sys
import time
from collections.abc import Mapping, Sequence
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import yaml
from execution_engine.adk.runtime import AgentRuntime
from execution_engine.agent import open_assembly
from execution_engine.config import Settings, get_settings
from execution_engine.recorder import InMemoryRecorder
from app.operating_model import digest
from app.operating_model.workflow import ROOT, build
from eval.scoring import CaseResult, Check, score
HERE = Path(__file__).resolve().parent
CASES = HERE / "datasets" / "cases.json"
RESULTS = HERE / "results"
logger = logging.getLogger(__name__)
def load_cases(wanted: Sequence[str] = ()) -> list[dict[str, Any]]:
cases: list[dict[str, Any]] = json.loads(CASES.read_text(encoding="utf-8"))
unknown = sorted(set(wanted) - {case["id"] for case in cases})
if unknown:
raise SystemExit(f"Unknown cases: {', '.join(unknown)}")
return [case for case in cases if not wanted or case["id"] in wanted]
def inputs(case: Mapping[str, Any]) -> dict[str, Any]:
"""What an Execution of the case is started with."""
given: dict[str, Any] = {}
if case.get("spec"):
given["scope"] = (CASES.parent / case["spec"]).read_text(encoding="utf-8")
if case.get("sources"):
given["sources"] = digest.read_folder(
(CASES.parent / case["sources"]).resolve()
)
if case.get("overlay"):
given["scope"] = yaml.safe_dump(case["overlay"])
return given
def _tokens(runtime: Any) -> dict[str, dict[str, int]]:
"""Tokens by role, when the runtime counts them."""
usage = getattr(runtime, "usage", None)
if not isinstance(usage, Mapping):
return {}
by_role: dict[str, dict[str, int]] = {}
for (role, _model), used in usage.items():
totals = by_role.setdefault(role, {})
for name, amount in vars(used).items():
if isinstance(amount, int):
totals[name] = totals.get(name, 0) + amount
return by_role
async def run_case(
case: Mapping[str, Any],
output_root: Path,
*,
settings: Settings | None = None,
runtime: AgentRuntime | None = None,
) -> CaseResult:
"""Run one case to the end and score it. `runtime` replaces the real one, for a
scripted run of the harness itself."""
settings = settings or get_settings()
workflow = build(ROOT)
started = time.monotonic()
async with open_assembly(
settings,
workflow,
root=ROOT,
runtime=runtime,
recorder=InMemoryRecorder(),
output_root=output_root,
) as assembly:
used = assembly.runtime
# A case is stopped at half again its time limit: a run that loops keeps
# spending until something stops it.
limit = float(case.get("max_minutes") or 60) * 1.5
try:
interaction = await asyncio.wait_for(
assembly.engine.start(
"", pre_approved=True, execution_id=case["id"], **inputs(case)
),
timeout=limit * 60,
)
except TimeoutError:
logger.error("Case %s stopped after %.0f minutes", case["id"], limit)
return CaseResult(
case=case["id"],
status="stopped",
checks=[Check("finished", False, f"stopped after {limit:.0f} minutes")],
minutes=round((time.monotonic() - started) / 60, 1),
tokens=_tokens(used),
)
except Exception as exc:
logger.exception("Case %s could not run", case["id"])
return CaseResult(
case=case["id"],
status="error",
checks=[Check("ran", False, f"{type(exc).__name__}: {exc}")],
minutes=round((time.monotonic() - started) / 60, 1),
)
minutes = round((time.monotonic() - started) / 60, 1)
state = interaction.state
checks, revisions, figures = score(
workflow,
state,
status=interaction.status,
max_revisions=int(case.get("max_revisions", 2)),
)
allowed = case.get("max_minutes")
if allowed:
checks.append(
Check(
f"finished within {allowed} minutes", minutes <= allowed, f"{minutes}"
)
)
return CaseResult(
case=case["id"],
status=interaction.status,
checks=checks,
minutes=minutes,
revisions=revisions,
tokens=_tokens(used),
figures=figures,
warnings=list(state.get("warnings") or []),
folder=str(
output_root
/ "deliverables"
/ "cases"
/ str(state.get("customer_case"))
/ "executions"
/ case["id"]
),
)
def report(results: Sequence[CaseResult]) -> str:
lines = []
for result in results:
lines.append(
f"{'PASS' if result.passed else 'FAIL'} {result.case} "
f"{result.status}, {result.minutes} min"
)
lines += [
f" {'ok ' if check.passed else 'FAIL'} {check.name}"
+ (f": {check.detail}" if check.detail else "")
for check in result.checks
]
if result.figures:
lines.append(
" " + ", ".join(f"{k}={v}" for k, v in result.figures.items())
)
lines += [f" warning: {warning}" for warning in result.warnings]
return "\n".join(lines)
async def main(arguments: Sequence[str]) -> int:
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
cases = load_cases(arguments)
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
output_root = RESULTS / stamp
output_root.mkdir(parents=True, exist_ok=True)
# An evaluation keeps nothing between runs and reads nobody's cloud.
settings = get_settings().model_copy(
update={"CHECKPOINTER": "memory", "MCP_ENABLED": False, "CLOUD_PROVIDER": "gcp"}
)
results = []
for case in cases:
logger.info("Running %s", case["id"])
results.append(await run_case(case, output_root, settings=settings))
(output_root / "results.json").write_text(
json.dumps([result.as_dict() for result in results], indent=2),
encoding="utf-8",
)
print(report(results))
print(f"\nWritten to {output_root}")
return 0 if all(result.passed for result in results) else 1
if __name__ == "__main__":
sys.exit(asyncio.run(main(sys.argv[1:])))

198
eval/scoring.py Normal file
View File

@@ -0,0 +1,198 @@
"""Checks applied to a finished Execution of the target operating model Workflow.
Every check here is structural: it reads the Execution's own data and asks whether
the chain holds and the result is what the Workflow promises. None judges whether
the operating model is a good one; that needs a person, or a judge calibrated
against one, and is not built.
"""
from __future__ import annotations
import json
from collections.abc import Mapping
from dataclasses import asdict, dataclass, field
from typing import Any
from execution_engine.workflows.workflow import Workflow
from app.operating_model import guide, scenarios, spec
from app.operating_model.names import (
GUIDE,
SCENARIOS_FILE,
STAFFING_FILE,
TARGET_OPERATING_MODEL,
VALIDATION_RESULTS,
VERDICTS_FILE,
)
@dataclass(frozen=True)
class Check:
name: str
passed: bool
detail: str = ""
@dataclass
class CaseResult:
case: str
status: str
checks: list[Check] = field(default_factory=list)
minutes: float = 0.0
revisions: dict[str, int] = field(default_factory=dict)
tokens: dict[str, dict[str, int]] = field(default_factory=dict)
figures: dict[str, Any] = field(default_factory=dict)
warnings: list[str] = field(default_factory=list)
folder: str = ""
@property
def passed(self) -> bool:
return all(check.passed for check in self.checks)
def as_dict(self) -> dict[str, Any]:
return {**asdict(self), "passed": self.passed}
def _attachment(deliverables: Mapping[str, Any], kind: str, name: str) -> Any:
text = (deliverables.get(kind) or {}).get("attachments", {}).get(name, "")
try:
return json.loads(text) if text else None
except ValueError:
return None
def score(
workflow: Workflow,
state: Mapping[str, Any],
*,
status: str,
max_revisions: int = 2,
) -> tuple[list[Check], dict[str, int], dict[str, Any]]:
"""The checks on a finished Execution, its Revisions, and its key figures."""
deliverables: Mapping[str, Any] = state.get("deliverables") or {}
ledger = (state.get("ledger") or {}).get("deliverables") or {}
revisions = {
kind: int(entry.get("revisions", 0))
for kind, entry in ledger.items()
if entry.get("revisions")
}
checks = [
Check("completed", status == "completed", status),
Check(
"every Deliverable was submitted",
set(deliverables) == set(workflow.deliverables),
", ".join(sorted(set(workflow.deliverables) - set(deliverables))) or "all",
),
]
# The chain, checked again on what was delivered: each Deliverable's data
# against the ones it builds on, as the engine checked it when it was submitted.
broken = []
for kind in workflow.deliverables:
if kind not in deliverables or kind not in workflow.data_checks:
continue
name, check = workflow.data_checks[kind]
attachments = deliverables[kind].get("attachments", {})
upstream = {
k: deliverables[k] for k in workflow.inputs_of(kind) if k in deliverables
}
broken += (
[f"{kind}: {fault}" for fault in check(attachments, upstream)]
if name in attachments
else [f"{kind}: {name} is missing"]
)
checks.append(Check("the chain holds", not broken, "; ".join(broken[:5])))
verdicts = _attachment(deliverables, VALIDATION_RESULTS, VERDICTS_FILE) or {}
given = [v for v in verdicts.get("checks") or [] if isinstance(v, dict)]
own = [v for v in given if not str(v.get("check", "")).startswith(scenarios.PREFIX)]
failed = [str(v.get("check")) for v in given if v.get("verdict") == "fail"]
checks.append(Check("no Check failed", not failed, ", ".join(failed)))
walked = _attachment(deliverables, VALIDATION_RESULTS, SCENARIOS_FILE) or []
checks.append(
Check(
"every scenario has a path",
bool(walked) and all(item.get("passes") for item in walked),
f"{sum(1 for i in walked if i.get('passes'))} of {len(walked)}",
)
)
content = str((deliverables.get(GUIDE) or {}).get("content", ""))
headings = [
line[3:].strip() for line in content.splitlines() if line.startswith("## ")
]
checks.append(
Check(
"the guide has its sections",
headings == list(guide.SECTIONS),
", ".join(headings),
)
)
summary = content.split(f"## {guide.SUMMARY}")[-1].split(
f"### {guide.KEY_FIGURES}"
)[0]
checks.append(
Check(
"the packager wrote a summary",
len(summary.split()) >= 30,
f"{len(summary.split())} words",
)
)
most = max(revisions.values(), default=0)
checks.append(
Check(
f"no Deliverable was revised more than {max_revisions} times",
most <= max_revisions,
", ".join(f"{kind} x {count}" for kind, count in revisions.items())
or "none",
)
)
scope = spec.agreed(deliverables)
cost = _attachment(deliverables, TARGET_OPERATING_MODEL, STAFFING_FILE) or {}
model = (
_attachment(deliverables, TARGET_OPERATING_MODEL, "target-operating-model.json")
or {}
)
figures = {
"items": len(spec.items(scope)),
"conditions": len(spec.conditions(scope)),
"operational_requirements": len(
_attachment(
deliverables,
"operational_requirements",
"operational-requirements.json",
)
or []
),
"capabilities": len(
_attachment(deliverables, "capability_model", "capabilities.json") or []
),
"custom_capabilities": sum(
1
for c in _attachment(deliverables, "capability_model", "capabilities.json")
or []
if c.get("catalogue") == "custom"
),
"parties": len(model.get("parties") or []),
"roles": len(model.get("roles") or []),
"checks": len(own),
"checks_not_verifiable": sum(
1 for v in own if v.get("verdict") == "not_verifiable"
),
"checks_waiting_on_findings": sum(
1 for v in own if v.get("verdict") == scenarios.WAITING
),
"proposed_additions": len(verdicts.get("proposed_additions") or []),
"scenarios": len(walked),
"findings": len(
_attachment(deliverables, TARGET_OPERATING_MODEL, "findings.json") or []
),
"business_outcomes": len(spec.business_outcomes(scope)),
"measures": len(model.get("measures") or []),
"people": cost.get("people"),
"cost_per_month": cost.get("cost_per_month"),
"currency": cost.get("currency"),
}
return checks, revisions, figures