feat: working TOM agent
This commit is contained in:
0
eval/__init__.py
Normal file
0
eval/__init__.py
Normal file
32
eval/datasets/cases.json
Normal file
32
eval/datasets/cases.json
Normal file
@@ -0,0 +1,32 @@
|
||||
[
|
||||
{
|
||||
"id": "case-01-landing-zone-shared",
|
||||
"what": "A written operating spec, complete enough to run with nobody to ask.",
|
||||
"spec": "specs/landing-zone-shared.yaml",
|
||||
"max_revisions": 2,
|
||||
"max_minutes": 40
|
||||
},
|
||||
{
|
||||
"id": "case-02-from-architecture",
|
||||
"what": "The solution architecture agent's landing-zone Execution, digested, with who operates it stated and the business outcomes left to the agent.",
|
||||
"sources": "../../tests/fixtures/architecture/case-08-landing-zone",
|
||||
"overlay": {
|
||||
"sourcing": {
|
||||
"model": "provider_operated",
|
||||
"delivery_countries": [
|
||||
"GB",
|
||||
"PL",
|
||||
"IN"
|
||||
]
|
||||
},
|
||||
"run": {
|
||||
"pre_approved": true
|
||||
},
|
||||
"intent": {
|
||||
"business_outcomes": "decide"
|
||||
}
|
||||
},
|
||||
"max_revisions": 3,
|
||||
"max_minutes": 60
|
||||
}
|
||||
]
|
||||
53
eval/datasets/specs/landing-zone-shared.yaml
Normal file
53
eval/datasets/specs/landing-zone-shared.yaml
Normal file
@@ -0,0 +1,53 @@
|
||||
# A fictional benchmark: a small landing zone run as a shared service. No customer's
|
||||
# data is in it. Complete enough to run with nobody to ask.
|
||||
metadata:
|
||||
customer_case: bench-landing-zone-shared
|
||||
engagement:
|
||||
kind: platform_foundation
|
||||
intent:
|
||||
statement: |
|
||||
We want our landing zone run as a shared service from go-live. A provider should
|
||||
respond to faults at every hour, and our own platform team keeps the final say
|
||||
over changes to the network.
|
||||
business_outcomes:
|
||||
- {id: BO-1, outcome: New payment products reach customers in weeks rather than months., target: "A change reaches production within a day of approval."}
|
||||
- {id: BO-2, outcome: Payments are never unavailable for long., target: "Restored within 4 hours."}
|
||||
- {id: BO-3, outcome: We keep the platform engineers we hire.}
|
||||
requirements:
|
||||
- {id: REQ-01, statement: Our platform team approves every change to the network.}
|
||||
- {id: REQ-02, statement: We receive a monthly report of service levels and cost.}
|
||||
solution:
|
||||
cloud_provider: gcp
|
||||
items:
|
||||
- {id: S-1, name: Hub network and interconnect, kind: network, product: Cloud Interconnect}
|
||||
- {id: S-2, name: Audit log store, kind: storage, product: Cloud Storage}
|
||||
- {id: S-3, name: Deployment pipeline, kind: pipeline, product: Cloud Build}
|
||||
- {id: S-4, name: Encryption keys, kind: keys, product: Cloud KMS}
|
||||
conditions:
|
||||
- {id: K-1, "on": [S-1], condition: The platform is available at least 99.9% of each month.}
|
||||
- {id: K-2, "on": [S-2], condition: Audit logs cannot be deleted for seven years.}
|
||||
- {id: K-3, "on": [S-4], condition: Encryption keys are rotated every 90 days.}
|
||||
build:
|
||||
duration_days: 36
|
||||
operations:
|
||||
support_hours: 24x7
|
||||
availability_hours: 24x7
|
||||
change_process: decide
|
||||
sourcing:
|
||||
model: shared
|
||||
delivery_countries: [GB, PL, IN]
|
||||
organisation:
|
||||
existing_teams:
|
||||
- {name: Platform team, party: customer, hours: business, country: GB}
|
||||
service_management:
|
||||
tooling: decide
|
||||
service_levels:
|
||||
priority_1_response_minutes: 15
|
||||
recovery_time_hours: 4
|
||||
compliance:
|
||||
frameworks: [DORA]
|
||||
commercial:
|
||||
currency: GBP
|
||||
monthly_run_budget: 60000
|
||||
run:
|
||||
pre_approved: true
|
||||
208
eval/operating_model_eval.py
Normal file
208
eval/operating_model_eval.py
Normal file
@@ -0,0 +1,208 @@
|
||||
"""Evaluate the target operating model agent on benchmark cases with real models.
|
||||
|
||||
Each case runs one pre-approved Execution through the full assembly (the ADK runtime
|
||||
and the agent's own Skills and role files), then scores it with eval/scoring.py.
|
||||
It costs model usage: run it deliberately.
|
||||
|
||||
Usage:
|
||||
GOOGLE_GENAI_USE_ENTERPRISE=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \\
|
||||
.venv/bin/python -m eval.operating_model_eval [case-id ...]
|
||||
|
||||
The model settings are read from the environment or from `.env`. With no case named,
|
||||
every case in eval/datasets/cases.json runs, one after another. Results are written
|
||||
under eval/results/<time>/: each Execution's files, and results.json. The exit code
|
||||
is non-zero when a case fails a check or runs over its time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from execution_engine.adk.runtime import AgentRuntime
|
||||
from execution_engine.agent import open_assembly
|
||||
from execution_engine.config import Settings, get_settings
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
|
||||
from app.operating_model import digest
|
||||
from app.operating_model.workflow import ROOT, build
|
||||
from eval.scoring import CaseResult, Check, score
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
CASES = HERE / "datasets" / "cases.json"
|
||||
RESULTS = HERE / "results"
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_cases(wanted: Sequence[str] = ()) -> list[dict[str, Any]]:
|
||||
cases: list[dict[str, Any]] = json.loads(CASES.read_text(encoding="utf-8"))
|
||||
unknown = sorted(set(wanted) - {case["id"] for case in cases})
|
||||
if unknown:
|
||||
raise SystemExit(f"Unknown cases: {', '.join(unknown)}")
|
||||
return [case for case in cases if not wanted or case["id"] in wanted]
|
||||
|
||||
|
||||
def inputs(case: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""What an Execution of the case is started with."""
|
||||
given: dict[str, Any] = {}
|
||||
if case.get("spec"):
|
||||
given["scope"] = (CASES.parent / case["spec"]).read_text(encoding="utf-8")
|
||||
if case.get("sources"):
|
||||
given["sources"] = digest.read_folder(
|
||||
(CASES.parent / case["sources"]).resolve()
|
||||
)
|
||||
if case.get("overlay"):
|
||||
given["scope"] = yaml.safe_dump(case["overlay"])
|
||||
return given
|
||||
|
||||
|
||||
def _tokens(runtime: Any) -> dict[str, dict[str, int]]:
|
||||
"""Tokens by role, when the runtime counts them."""
|
||||
usage = getattr(runtime, "usage", None)
|
||||
if not isinstance(usage, Mapping):
|
||||
return {}
|
||||
by_role: dict[str, dict[str, int]] = {}
|
||||
for (role, _model), used in usage.items():
|
||||
totals = by_role.setdefault(role, {})
|
||||
for name, amount in vars(used).items():
|
||||
if isinstance(amount, int):
|
||||
totals[name] = totals.get(name, 0) + amount
|
||||
return by_role
|
||||
|
||||
|
||||
async def run_case(
|
||||
case: Mapping[str, Any],
|
||||
output_root: Path,
|
||||
*,
|
||||
settings: Settings | None = None,
|
||||
runtime: AgentRuntime | None = None,
|
||||
) -> CaseResult:
|
||||
"""Run one case to the end and score it. `runtime` replaces the real one, for a
|
||||
scripted run of the harness itself."""
|
||||
settings = settings or get_settings()
|
||||
workflow = build(ROOT)
|
||||
started = time.monotonic()
|
||||
async with open_assembly(
|
||||
settings,
|
||||
workflow,
|
||||
root=ROOT,
|
||||
runtime=runtime,
|
||||
recorder=InMemoryRecorder(),
|
||||
output_root=output_root,
|
||||
) as assembly:
|
||||
used = assembly.runtime
|
||||
# A case is stopped at half again its time limit: a run that loops keeps
|
||||
# spending until something stops it.
|
||||
limit = float(case.get("max_minutes") or 60) * 1.5
|
||||
try:
|
||||
interaction = await asyncio.wait_for(
|
||||
assembly.engine.start(
|
||||
"", pre_approved=True, execution_id=case["id"], **inputs(case)
|
||||
),
|
||||
timeout=limit * 60,
|
||||
)
|
||||
except TimeoutError:
|
||||
logger.error("Case %s stopped after %.0f minutes", case["id"], limit)
|
||||
return CaseResult(
|
||||
case=case["id"],
|
||||
status="stopped",
|
||||
checks=[Check("finished", False, f"stopped after {limit:.0f} minutes")],
|
||||
minutes=round((time.monotonic() - started) / 60, 1),
|
||||
tokens=_tokens(used),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception("Case %s could not run", case["id"])
|
||||
return CaseResult(
|
||||
case=case["id"],
|
||||
status="error",
|
||||
checks=[Check("ran", False, f"{type(exc).__name__}: {exc}")],
|
||||
minutes=round((time.monotonic() - started) / 60, 1),
|
||||
)
|
||||
minutes = round((time.monotonic() - started) / 60, 1)
|
||||
state = interaction.state
|
||||
checks, revisions, figures = score(
|
||||
workflow,
|
||||
state,
|
||||
status=interaction.status,
|
||||
max_revisions=int(case.get("max_revisions", 2)),
|
||||
)
|
||||
allowed = case.get("max_minutes")
|
||||
if allowed:
|
||||
checks.append(
|
||||
Check(
|
||||
f"finished within {allowed} minutes", minutes <= allowed, f"{minutes}"
|
||||
)
|
||||
)
|
||||
return CaseResult(
|
||||
case=case["id"],
|
||||
status=interaction.status,
|
||||
checks=checks,
|
||||
minutes=minutes,
|
||||
revisions=revisions,
|
||||
tokens=_tokens(used),
|
||||
figures=figures,
|
||||
warnings=list(state.get("warnings") or []),
|
||||
folder=str(
|
||||
output_root
|
||||
/ "deliverables"
|
||||
/ "cases"
|
||||
/ str(state.get("customer_case"))
|
||||
/ "executions"
|
||||
/ case["id"]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def report(results: Sequence[CaseResult]) -> str:
|
||||
lines = []
|
||||
for result in results:
|
||||
lines.append(
|
||||
f"{'PASS' if result.passed else 'FAIL'} {result.case} "
|
||||
f"{result.status}, {result.minutes} min"
|
||||
)
|
||||
lines += [
|
||||
f" {'ok ' if check.passed else 'FAIL'} {check.name}"
|
||||
+ (f": {check.detail}" if check.detail else "")
|
||||
for check in result.checks
|
||||
]
|
||||
if result.figures:
|
||||
lines.append(
|
||||
" " + ", ".join(f"{k}={v}" for k, v in result.figures.items())
|
||||
)
|
||||
lines += [f" warning: {warning}" for warning in result.warnings]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
async def main(arguments: Sequence[str]) -> int:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
|
||||
cases = load_cases(arguments)
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
output_root = RESULTS / stamp
|
||||
output_root.mkdir(parents=True, exist_ok=True)
|
||||
# An evaluation keeps nothing between runs and reads nobody's cloud.
|
||||
settings = get_settings().model_copy(
|
||||
update={"CHECKPOINTER": "memory", "MCP_ENABLED": False, "CLOUD_PROVIDER": "gcp"}
|
||||
)
|
||||
results = []
|
||||
for case in cases:
|
||||
logger.info("Running %s", case["id"])
|
||||
results.append(await run_case(case, output_root, settings=settings))
|
||||
(output_root / "results.json").write_text(
|
||||
json.dumps([result.as_dict() for result in results], indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(report(results))
|
||||
print(f"\nWritten to {output_root}")
|
||||
return 0 if all(result.passed for result in results) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main(sys.argv[1:])))
|
||||
198
eval/scoring.py
Normal file
198
eval/scoring.py
Normal file
@@ -0,0 +1,198 @@
|
||||
"""Checks applied to a finished Execution of the target operating model Workflow.
|
||||
|
||||
Every check here is structural: it reads the Execution's own data and asks whether
|
||||
the chain holds and the result is what the Workflow promises. None judges whether
|
||||
the operating model is a good one; that needs a person, or a judge calibrated
|
||||
against one, and is not built.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from execution_engine.workflows.workflow import Workflow
|
||||
|
||||
from app.operating_model import guide, scenarios, spec
|
||||
from app.operating_model.names import (
|
||||
GUIDE,
|
||||
SCENARIOS_FILE,
|
||||
STAFFING_FILE,
|
||||
TARGET_OPERATING_MODEL,
|
||||
VALIDATION_RESULTS,
|
||||
VERDICTS_FILE,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Check:
|
||||
name: str
|
||||
passed: bool
|
||||
detail: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class CaseResult:
|
||||
case: str
|
||||
status: str
|
||||
checks: list[Check] = field(default_factory=list)
|
||||
minutes: float = 0.0
|
||||
revisions: dict[str, int] = field(default_factory=dict)
|
||||
tokens: dict[str, dict[str, int]] = field(default_factory=dict)
|
||||
figures: dict[str, Any] = field(default_factory=dict)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
folder: str = ""
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
return all(check.passed for check in self.checks)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {**asdict(self), "passed": self.passed}
|
||||
|
||||
|
||||
def _attachment(deliverables: Mapping[str, Any], kind: str, name: str) -> Any:
|
||||
text = (deliverables.get(kind) or {}).get("attachments", {}).get(name, "")
|
||||
try:
|
||||
return json.loads(text) if text else None
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def score(
|
||||
workflow: Workflow,
|
||||
state: Mapping[str, Any],
|
||||
*,
|
||||
status: str,
|
||||
max_revisions: int = 2,
|
||||
) -> tuple[list[Check], dict[str, int], dict[str, Any]]:
|
||||
"""The checks on a finished Execution, its Revisions, and its key figures."""
|
||||
deliverables: Mapping[str, Any] = state.get("deliverables") or {}
|
||||
ledger = (state.get("ledger") or {}).get("deliverables") or {}
|
||||
revisions = {
|
||||
kind: int(entry.get("revisions", 0))
|
||||
for kind, entry in ledger.items()
|
||||
if entry.get("revisions")
|
||||
}
|
||||
checks = [
|
||||
Check("completed", status == "completed", status),
|
||||
Check(
|
||||
"every Deliverable was submitted",
|
||||
set(deliverables) == set(workflow.deliverables),
|
||||
", ".join(sorted(set(workflow.deliverables) - set(deliverables))) or "all",
|
||||
),
|
||||
]
|
||||
|
||||
# The chain, checked again on what was delivered: each Deliverable's data
|
||||
# against the ones it builds on, as the engine checked it when it was submitted.
|
||||
broken = []
|
||||
for kind in workflow.deliverables:
|
||||
if kind not in deliverables or kind not in workflow.data_checks:
|
||||
continue
|
||||
name, check = workflow.data_checks[kind]
|
||||
attachments = deliverables[kind].get("attachments", {})
|
||||
upstream = {
|
||||
k: deliverables[k] for k in workflow.inputs_of(kind) if k in deliverables
|
||||
}
|
||||
broken += (
|
||||
[f"{kind}: {fault}" for fault in check(attachments, upstream)]
|
||||
if name in attachments
|
||||
else [f"{kind}: {name} is missing"]
|
||||
)
|
||||
checks.append(Check("the chain holds", not broken, "; ".join(broken[:5])))
|
||||
|
||||
verdicts = _attachment(deliverables, VALIDATION_RESULTS, VERDICTS_FILE) or {}
|
||||
given = [v for v in verdicts.get("checks") or [] if isinstance(v, dict)]
|
||||
own = [v for v in given if not str(v.get("check", "")).startswith(scenarios.PREFIX)]
|
||||
failed = [str(v.get("check")) for v in given if v.get("verdict") == "fail"]
|
||||
checks.append(Check("no Check failed", not failed, ", ".join(failed)))
|
||||
walked = _attachment(deliverables, VALIDATION_RESULTS, SCENARIOS_FILE) or []
|
||||
checks.append(
|
||||
Check(
|
||||
"every scenario has a path",
|
||||
bool(walked) and all(item.get("passes") for item in walked),
|
||||
f"{sum(1 for i in walked if i.get('passes'))} of {len(walked)}",
|
||||
)
|
||||
)
|
||||
|
||||
content = str((deliverables.get(GUIDE) or {}).get("content", ""))
|
||||
headings = [
|
||||
line[3:].strip() for line in content.splitlines() if line.startswith("## ")
|
||||
]
|
||||
checks.append(
|
||||
Check(
|
||||
"the guide has its sections",
|
||||
headings == list(guide.SECTIONS),
|
||||
", ".join(headings),
|
||||
)
|
||||
)
|
||||
summary = content.split(f"## {guide.SUMMARY}")[-1].split(
|
||||
f"### {guide.KEY_FIGURES}"
|
||||
)[0]
|
||||
checks.append(
|
||||
Check(
|
||||
"the packager wrote a summary",
|
||||
len(summary.split()) >= 30,
|
||||
f"{len(summary.split())} words",
|
||||
)
|
||||
)
|
||||
|
||||
most = max(revisions.values(), default=0)
|
||||
checks.append(
|
||||
Check(
|
||||
f"no Deliverable was revised more than {max_revisions} times",
|
||||
most <= max_revisions,
|
||||
", ".join(f"{kind} x {count}" for kind, count in revisions.items())
|
||||
or "none",
|
||||
)
|
||||
)
|
||||
|
||||
scope = spec.agreed(deliverables)
|
||||
cost = _attachment(deliverables, TARGET_OPERATING_MODEL, STAFFING_FILE) or {}
|
||||
model = (
|
||||
_attachment(deliverables, TARGET_OPERATING_MODEL, "target-operating-model.json")
|
||||
or {}
|
||||
)
|
||||
figures = {
|
||||
"items": len(spec.items(scope)),
|
||||
"conditions": len(spec.conditions(scope)),
|
||||
"operational_requirements": len(
|
||||
_attachment(
|
||||
deliverables,
|
||||
"operational_requirements",
|
||||
"operational-requirements.json",
|
||||
)
|
||||
or []
|
||||
),
|
||||
"capabilities": len(
|
||||
_attachment(deliverables, "capability_model", "capabilities.json") or []
|
||||
),
|
||||
"custom_capabilities": sum(
|
||||
1
|
||||
for c in _attachment(deliverables, "capability_model", "capabilities.json")
|
||||
or []
|
||||
if c.get("catalogue") == "custom"
|
||||
),
|
||||
"parties": len(model.get("parties") or []),
|
||||
"roles": len(model.get("roles") or []),
|
||||
"checks": len(own),
|
||||
"checks_not_verifiable": sum(
|
||||
1 for v in own if v.get("verdict") == "not_verifiable"
|
||||
),
|
||||
"checks_waiting_on_findings": sum(
|
||||
1 for v in own if v.get("verdict") == scenarios.WAITING
|
||||
),
|
||||
"proposed_additions": len(verdicts.get("proposed_additions") or []),
|
||||
"scenarios": len(walked),
|
||||
"findings": len(
|
||||
_attachment(deliverables, TARGET_OPERATING_MODEL, "findings.json") or []
|
||||
),
|
||||
"business_outcomes": len(spec.business_outcomes(scope)),
|
||||
"measures": len(model.get("measures") or []),
|
||||
"people": cost.get("people"),
|
||||
"cost_per_month": cost.get("cost_per_month"),
|
||||
"currency": cost.get("currency"),
|
||||
}
|
||||
return checks, revisions, figures
|
||||
Reference in New Issue
Block a user