feat: establish the Target architecture as a graph before code

Design is now approved before any infrastructure code is written (ADR-0005).
A new target-architect Specialist produces one graph of Elements,
Relationships and Checks; the diagram is rendered from it, deployment
guidance must account for every Check, and validation gives a verdict per
Check. Requirements, components, products and recommendations carry ids in
data files, and a submission with a gap in the Requirements chain is refused.

Also in this commit:
- Customer cases group Executions, with an intent history per case
- lesson candidates are captured when an Execution ends
- evaluation gains chain, verdict, judged and interactive checks, and
  per-case thresholds
- Terraform providers are cached between checks
- the requirements analyst proposes defaults instead of asking
- the legacy-baseline case and the checks that only read it are removed
- ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed
- plans for iterations 4 to 7 under .agents/plans/

Unproven with a real model: no evaluation run has completed on this design.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-07 13:59:09 +01:00
parent 7ee49ab352
commit e9246828b3
586 changed files with 5166 additions and 22432 deletions

View File

@@ -6,7 +6,12 @@ eval/scoring.py. Costs model usage: run deliberately.
Usage:
GOOGLE_GENAI_USE_VERTEXAI=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \
.venv/bin/python -m eval.solution_eval [case-id ...]
.venv/bin/python -m eval.solution_eval [--judge] [case-id ...]
--judge adds the judged checks (pillar checklist coverage, recommendation quality),
using the provider bundle's default model unless EVAL_JUDGE_MODEL names another.
A case marked "interactive" runs with a scripted user in the loop. The exit code is
non-zero when a case fails or exceeds one of its thresholds.
"""
from __future__ import annotations
@@ -14,6 +19,7 @@ from __future__ import annotations
import asyncio
import json
import logging
import os
import sys
import time
from collections import Counter
@@ -24,18 +30,25 @@ from pathlib import Path
from typing import Any
from app.adk.runtime import AgentRuntime, AgentTask, Outcome, RoleSpec, ToolSpec
from app.agent import open_assembly
from app.agent import ExecutionEngine, Interaction, open_assembly
from app.config import Settings, get_settings
from app.nodes.execution_nodes import automatic_reply
from app.recorder import InMemoryRecorder
from app.states.execution_state import execution_dir
from app.tools.local import TerraformRunner
from eval.judge import GeminiJudge, Judge
from eval.scoring import (
CaseResult,
CaseSpec,
checklist_items,
load_cases,
regressions,
score_execution,
template_headings,
)
GEMINI_PREFIX = "gemini/"
MAX_USER_TURNS = 80 # an interactive case stops being answered after this many prompts
ROOT = Path(__file__).resolve().parent
CASES_PATH = ROOT / "datasets" / "solution_cases.json"
RESULTS_DIR = ROOT / "results"
@@ -96,7 +109,33 @@ def token_report(usage: dict[tuple[str, str], Any]) -> dict[str, Any]:
}
async def run_case(case: CaseSpec, settings: Settings, run_dir: Path) -> CaseResult:
async def run_with_scripted_user(
engine: ExecutionEngine,
case: CaseSpec,
execution_id: str,
max_user_turns: int = MAX_USER_TURNS,
) -> Interaction:
"""Run an interactive Execution, answering each prompt as the case scripts it:
Approvals are granted, cloud reads denied, and questions answered by pattern."""
interaction = await engine.start(
case.workflow_request,
customer_case=case.id,
allow_cloud_read=False,
execution_id=execution_id,
)
for _ in range(max_user_turns):
if interaction.status != "awaiting_user" or interaction.prompt is None:
break
reply = automatic_reply(
interaction.prompt, [list(pair) for pair in case.scripted_answers]
)
interaction = await engine.resume(execution_id, reply)
return interaction
async def run_case(
case: CaseSpec, settings: Settings, run_dir: Path, judge: Judge | None = None
) -> CaseResult:
from app.adk.adk_runtime import AdkAgentRuntime
from app.providers import load_provider
@@ -111,48 +150,84 @@ async def run_case(case: CaseSpec, settings: Settings, run_dir: Path) -> CaseRes
async with open_assembly(
settings, runtime=runtime, recorder=recorder, output_root=run_dir
) as assembly:
interaction = await assembly.engine.start(
case.workflow_request,
pre_approved=True,
allow_cloud_read=False,
execution_id=f"{case.id}-{datetime.now(UTC):%H%M%S}",
scripted_answers=case.scripted_answers,
)
execution_id = f"{case.id}-{datetime.now(UTC):%H%M%S}"
if case.interactive:
interaction = await run_with_scripted_user(
assembly.engine, case, execution_id
)
else:
interaction = await assembly.engine.start(
case.workflow_request,
customer_case=case.id,
pre_approved=True,
allow_cloud_read=False,
execution_id=execution_id,
scripted_answers=case.scripted_answers,
)
elapsed = time.monotonic() - started
runner = TerraformRunner(
binary=settings.TERRAFORM_BINARY,
timeout_seconds=settings.TERRAFORM_TIMEOUT_SECONDS,
plugin_cache_dir=settings.TF_PLUGIN_CACHE_DIR,
plugin_cache_dir=settings.terraform_plugin_cache(),
)
result = await score_execution(
case, interaction, template_headings(bundle.output_template), runner.validate
case,
interaction,
template_headings(bundle.output_template),
runner.validate,
judge=judge,
pillar_checklists={
pillar: checklist_items(bundle.skills[skill].path / "SKILL.md")
for pillar, skill in bundle.pillar_skills.items()
},
timeline=runtime.timeline,
)
ledger = interaction.state.get("ledger", {})
tokens = token_report(adk_runtime.usage)
revisions = {
kind: item["revisions"]
for kind, item in ledger.get("deliverables", {}).items()
if item["revisions"]
}
details = {
"execution_id": interaction.execution_id,
"interactive": case.interactive,
"regressions": regressions(
case,
{
"revisions": sum(revisions.values()),
"seconds": elapsed,
"tokens": tokens["total"].get("total"),
},
),
"seconds": round(elapsed, 1),
"first_pass_seconds": first_pass_seconds(runtime.timeline),
"orchestrator_turns": interaction.state.get("turns"),
"tokens": token_report(adk_runtime.usage),
"tokens": tokens,
"timeline": runtime.timeline,
"agent_runs": dict(runtime.runs),
"outcomes": dict(runtime.outcomes),
"revisions": {
kind: item["revisions"]
for kind, item in ledger.get("deliverables", {}).items()
if item["revisions"]
},
"revisions": revisions,
"warnings": interaction.state.get("warnings", []),
"conversation": interaction.state.get("conversation", []),
"output_dir": str(
run_dir / "deliverables" / "executions" / interaction.execution_id
),
"output_dir": str(execution_dir(run_dir, interaction.execution_id, case.id)),
}
return CaseResult(result.case_id, result.status, result.checks, details)
async def main(case_ids: Sequence[str]) -> int:
def make_judge(settings: Settings) -> Judge:
from app.providers import load_provider
model = os.environ.get("EVAL_JUDGE_MODEL") or load_provider(
settings.CLOUD_PROVIDER
).models.default.removeprefix(GEMINI_PREFIX)
return GeminiJudge(model)
async def main(arguments: Sequence[str]) -> int:
use_judge = "--judge" in arguments
case_ids = [argument for argument in arguments if argument != "--judge"]
# Progress (Orchestrator turns, Specialist runs, tool calls, Ledger events) at
# INFO; everything else, including ADK and HTTP clients, at LOG_LEVEL.
logging.basicConfig(
@@ -168,11 +243,12 @@ async def main(case_ids: Sequence[str]) -> int:
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
run_dir = RESULTS_DIR / stamp
run_dir.mkdir(parents=True, exist_ok=True)
judge = make_judge(settings) if use_judge else None
results: list[dict[str, Any]] = []
for case in cases:
print(f"== {case.id}: running", flush=True)
try:
result = await run_case(case, settings, run_dir)
result = await run_case(case, settings, run_dir, judge)
except Exception as exc: # noqa: BLE001 - record the failure and continue
print(f" ERROR {type(exc).__name__}: {exc}", flush=True)
results.append(
@@ -192,6 +268,8 @@ async def main(case_ids: Sequence[str]) -> int:
f"revisions {details['revisions'] or 'none'}",
flush=True,
)
for regression in details["regressions"]:
print(f" REGRESSION {regression}", flush=True)
for model, counts in details["tokens"]["by_model"].items():
print(f" tokens {model}: {counts}", flush=True)
print(f" tokens total: {details['tokens']['total']}", flush=True)
@@ -209,8 +287,12 @@ async def main(case_ids: Sequence[str]) -> int:
report = run_dir / "report.json"
report.write_text(json.dumps(results, indent=2, default=str), encoding="utf-8")
passed = sum(1 for r in results if r.get("passed"))
print(f"\n{passed}/{len(results)} cases passed. Report: {report}")
return 0 if passed == len(results) else 1
regressed = sum(1 for r in results if r.get("details", {}).get("regressions"))
print(
f"\n{passed}/{len(results)} cases passed, {regressed} regressed. "
f"Report: {report}"
)
return 0 if passed == len(results) and not regressed else 1
if __name__ == "__main__":