feat: establish the Target architecture as a graph before code
Design is now approved before any infrastructure code is written (ADR-0005). A new target-architect Specialist produces one graph of Elements, Relationships and Checks; the diagram is rendered from it, deployment guidance must account for every Check, and validation gives a verdict per Check. Requirements, components, products and recommendations carry ids in data files, and a submission with a gap in the Requirements chain is refused. Also in this commit: - Customer cases group Executions, with an intent history per case - lesson candidates are captured when an Execution ends - evaluation gains chain, verdict, judged and interactive checks, and per-case thresholds - Terraform providers are cached between checks - the requirements analyst proposes defaults instead of asking - the legacy-baseline case and the checks that only read it are removed - ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed - plans for iterations 4 to 7 under .agents/plans/ Unproven with a real model: no evaluation run has completed on this design. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -6,7 +6,12 @@ eval/scoring.py. Costs model usage: run deliberately.
|
||||
|
||||
Usage:
|
||||
GOOGLE_GENAI_USE_VERTEXAI=true GOOGLE_CLOUD_PROJECT=<project> GOOGLE_CLOUD_LOCATION=global \
|
||||
.venv/bin/python -m eval.solution_eval [case-id ...]
|
||||
.venv/bin/python -m eval.solution_eval [--judge] [case-id ...]
|
||||
|
||||
--judge adds the judged checks (pillar checklist coverage, recommendation quality),
|
||||
using the provider bundle's default model unless EVAL_JUDGE_MODEL names another.
|
||||
A case marked "interactive" runs with a scripted user in the loop. The exit code is
|
||||
non-zero when a case fails or exceeds one of its thresholds.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -14,6 +19,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
@@ -24,18 +30,25 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.adk.runtime import AgentRuntime, AgentTask, Outcome, RoleSpec, ToolSpec
|
||||
from app.agent import open_assembly
|
||||
from app.agent import ExecutionEngine, Interaction, open_assembly
|
||||
from app.config import Settings, get_settings
|
||||
from app.nodes.execution_nodes import automatic_reply
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.states.execution_state import execution_dir
|
||||
from app.tools.local import TerraformRunner
|
||||
from eval.judge import GeminiJudge, Judge
|
||||
from eval.scoring import (
|
||||
CaseResult,
|
||||
CaseSpec,
|
||||
checklist_items,
|
||||
load_cases,
|
||||
regressions,
|
||||
score_execution,
|
||||
template_headings,
|
||||
)
|
||||
|
||||
GEMINI_PREFIX = "gemini/"
|
||||
MAX_USER_TURNS = 80 # an interactive case stops being answered after this many prompts
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
CASES_PATH = ROOT / "datasets" / "solution_cases.json"
|
||||
RESULTS_DIR = ROOT / "results"
|
||||
@@ -96,7 +109,33 @@ def token_report(usage: dict[tuple[str, str], Any]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
async def run_case(case: CaseSpec, settings: Settings, run_dir: Path) -> CaseResult:
|
||||
async def run_with_scripted_user(
|
||||
engine: ExecutionEngine,
|
||||
case: CaseSpec,
|
||||
execution_id: str,
|
||||
max_user_turns: int = MAX_USER_TURNS,
|
||||
) -> Interaction:
|
||||
"""Run an interactive Execution, answering each prompt as the case scripts it:
|
||||
Approvals are granted, cloud reads denied, and questions answered by pattern."""
|
||||
interaction = await engine.start(
|
||||
case.workflow_request,
|
||||
customer_case=case.id,
|
||||
allow_cloud_read=False,
|
||||
execution_id=execution_id,
|
||||
)
|
||||
for _ in range(max_user_turns):
|
||||
if interaction.status != "awaiting_user" or interaction.prompt is None:
|
||||
break
|
||||
reply = automatic_reply(
|
||||
interaction.prompt, [list(pair) for pair in case.scripted_answers]
|
||||
)
|
||||
interaction = await engine.resume(execution_id, reply)
|
||||
return interaction
|
||||
|
||||
|
||||
async def run_case(
|
||||
case: CaseSpec, settings: Settings, run_dir: Path, judge: Judge | None = None
|
||||
) -> CaseResult:
|
||||
from app.adk.adk_runtime import AdkAgentRuntime
|
||||
from app.providers import load_provider
|
||||
|
||||
@@ -111,48 +150,84 @@ async def run_case(case: CaseSpec, settings: Settings, run_dir: Path) -> CaseRes
|
||||
async with open_assembly(
|
||||
settings, runtime=runtime, recorder=recorder, output_root=run_dir
|
||||
) as assembly:
|
||||
interaction = await assembly.engine.start(
|
||||
case.workflow_request,
|
||||
pre_approved=True,
|
||||
allow_cloud_read=False,
|
||||
execution_id=f"{case.id}-{datetime.now(UTC):%H%M%S}",
|
||||
scripted_answers=case.scripted_answers,
|
||||
)
|
||||
execution_id = f"{case.id}-{datetime.now(UTC):%H%M%S}"
|
||||
if case.interactive:
|
||||
interaction = await run_with_scripted_user(
|
||||
assembly.engine, case, execution_id
|
||||
)
|
||||
else:
|
||||
interaction = await assembly.engine.start(
|
||||
case.workflow_request,
|
||||
customer_case=case.id,
|
||||
pre_approved=True,
|
||||
allow_cloud_read=False,
|
||||
execution_id=execution_id,
|
||||
scripted_answers=case.scripted_answers,
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
runner = TerraformRunner(
|
||||
binary=settings.TERRAFORM_BINARY,
|
||||
timeout_seconds=settings.TERRAFORM_TIMEOUT_SECONDS,
|
||||
plugin_cache_dir=settings.TF_PLUGIN_CACHE_DIR,
|
||||
plugin_cache_dir=settings.terraform_plugin_cache(),
|
||||
)
|
||||
result = await score_execution(
|
||||
case, interaction, template_headings(bundle.output_template), runner.validate
|
||||
case,
|
||||
interaction,
|
||||
template_headings(bundle.output_template),
|
||||
runner.validate,
|
||||
judge=judge,
|
||||
pillar_checklists={
|
||||
pillar: checklist_items(bundle.skills[skill].path / "SKILL.md")
|
||||
for pillar, skill in bundle.pillar_skills.items()
|
||||
},
|
||||
timeline=runtime.timeline,
|
||||
)
|
||||
ledger = interaction.state.get("ledger", {})
|
||||
tokens = token_report(adk_runtime.usage)
|
||||
revisions = {
|
||||
kind: item["revisions"]
|
||||
for kind, item in ledger.get("deliverables", {}).items()
|
||||
if item["revisions"]
|
||||
}
|
||||
details = {
|
||||
"execution_id": interaction.execution_id,
|
||||
"interactive": case.interactive,
|
||||
"regressions": regressions(
|
||||
case,
|
||||
{
|
||||
"revisions": sum(revisions.values()),
|
||||
"seconds": elapsed,
|
||||
"tokens": tokens["total"].get("total"),
|
||||
},
|
||||
),
|
||||
"seconds": round(elapsed, 1),
|
||||
"first_pass_seconds": first_pass_seconds(runtime.timeline),
|
||||
"orchestrator_turns": interaction.state.get("turns"),
|
||||
"tokens": token_report(adk_runtime.usage),
|
||||
"tokens": tokens,
|
||||
"timeline": runtime.timeline,
|
||||
"agent_runs": dict(runtime.runs),
|
||||
"outcomes": dict(runtime.outcomes),
|
||||
"revisions": {
|
||||
kind: item["revisions"]
|
||||
for kind, item in ledger.get("deliverables", {}).items()
|
||||
if item["revisions"]
|
||||
},
|
||||
"revisions": revisions,
|
||||
"warnings": interaction.state.get("warnings", []),
|
||||
"conversation": interaction.state.get("conversation", []),
|
||||
"output_dir": str(
|
||||
run_dir / "deliverables" / "executions" / interaction.execution_id
|
||||
),
|
||||
"output_dir": str(execution_dir(run_dir, interaction.execution_id, case.id)),
|
||||
}
|
||||
return CaseResult(result.case_id, result.status, result.checks, details)
|
||||
|
||||
|
||||
async def main(case_ids: Sequence[str]) -> int:
|
||||
def make_judge(settings: Settings) -> Judge:
|
||||
from app.providers import load_provider
|
||||
|
||||
model = os.environ.get("EVAL_JUDGE_MODEL") or load_provider(
|
||||
settings.CLOUD_PROVIDER
|
||||
).models.default.removeprefix(GEMINI_PREFIX)
|
||||
return GeminiJudge(model)
|
||||
|
||||
|
||||
async def main(arguments: Sequence[str]) -> int:
|
||||
use_judge = "--judge" in arguments
|
||||
case_ids = [argument for argument in arguments if argument != "--judge"]
|
||||
# Progress (Orchestrator turns, Specialist runs, tool calls, Ledger events) at
|
||||
# INFO; everything else, including ADK and HTTP clients, at LOG_LEVEL.
|
||||
logging.basicConfig(
|
||||
@@ -168,11 +243,12 @@ async def main(case_ids: Sequence[str]) -> int:
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
run_dir = RESULTS_DIR / stamp
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
judge = make_judge(settings) if use_judge else None
|
||||
results: list[dict[str, Any]] = []
|
||||
for case in cases:
|
||||
print(f"== {case.id}: running", flush=True)
|
||||
try:
|
||||
result = await run_case(case, settings, run_dir)
|
||||
result = await run_case(case, settings, run_dir, judge)
|
||||
except Exception as exc: # noqa: BLE001 - record the failure and continue
|
||||
print(f" ERROR {type(exc).__name__}: {exc}", flush=True)
|
||||
results.append(
|
||||
@@ -192,6 +268,8 @@ async def main(case_ids: Sequence[str]) -> int:
|
||||
f"revisions {details['revisions'] or 'none'}",
|
||||
flush=True,
|
||||
)
|
||||
for regression in details["regressions"]:
|
||||
print(f" REGRESSION {regression}", flush=True)
|
||||
for model, counts in details["tokens"]["by_model"].items():
|
||||
print(f" tokens {model}: {counts}", flush=True)
|
||||
print(f" tokens total: {details['tokens']['total']}", flush=True)
|
||||
@@ -209,8 +287,12 @@ async def main(case_ids: Sequence[str]) -> int:
|
||||
report = run_dir / "report.json"
|
||||
report.write_text(json.dumps(results, indent=2, default=str), encoding="utf-8")
|
||||
passed = sum(1 for r in results if r.get("passed"))
|
||||
print(f"\n{passed}/{len(results)} cases passed. Report: {report}")
|
||||
return 0 if passed == len(results) else 1
|
||||
regressed = sum(1 for r in results if r.get("details", {}).get("regressions"))
|
||||
print(
|
||||
f"\n{passed}/{len(results)} cases passed, {regressed} regressed. "
|
||||
f"Report: {report}"
|
||||
)
|
||||
return 0 if passed == len(results) and not regressed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user