feat: establish the Target architecture as a graph before code

Design is now approved before any infrastructure code is written (ADR-0005).
A new target-architect Specialist produces one graph of Elements,
Relationships and Checks; the diagram is rendered from it, deployment
guidance must account for every Check, and validation gives a verdict per
Check. Requirements, components, products and recommendations carry ids in
data files, and a submission with a gap in the Requirements chain is refused.

Also in this commit:
- Customer cases group Executions, with an intent history per case
- lesson candidates are captured when an Execution ends
- evaluation gains chain, verdict, judged and interactive checks, and
  per-case thresholds
- Terraform providers are cached between checks
- the requirements analyst proposes defaults instead of asking
- the legacy-baseline case and the checks that only read it are removed
- ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed
- plans for iterations 4 to 7 under .agents/plans/

Unproven with a real model: no evaluation run has completed on this design.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-07 13:59:09 +01:00
parent 7ee49ab352
commit e9246828b3
586 changed files with 5166 additions and 22432 deletions

View File

@@ -2,8 +2,10 @@
from pathlib import Path
from app.adk.fake_runtime import SAMPLE_DATA
from app.adk.runtime import ToolResult
from app.agent import Interaction
from app.workflows.target_architecture import parse_graph, render_mermaid
from eval.scoring import CaseSpec, score_execution, template_headings
TEMPLATE = (
@@ -25,11 +27,18 @@ def guide(headings=HEADINGS) -> str:
return "\n".join(f"## {heading}\nContent." for heading in headings)
GRAPH = SAMPLE_DATA["target-architecture.json"]
def execution(status="completed", **overrides) -> Interaction:
deliverables = {
"requirements": {
"content": "## Functional requirements\nAccept webhooks.\n## Current state\nNone.",
"attachments": {},
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]},
},
"technical_decomposition": {
"content": "## Components\nIngress.",
"attachments": {"components.json": SAMPLE_DATA["components.json"]},
},
"product_selection": {
"content": (
@@ -39,12 +48,19 @@ def execution(status="completed", **overrides) -> Interaction:
"**Pros**: control **Cons**: toil |\n"
"| Queue | Pub/Sub | ... | Cloud Tasks | ... |\n| Archive | Cloud Storage | ... |"
),
"attachments": {},
"attachments": {"products.json": SAMPLE_DATA["products.json"]},
},
"architecture_diagram": {
"content": "diagram",
"design_recommendations": {
"content": "## Reliability\nUse two zones because NFR-1 asks for 99.9%.",
"attachments": {
"architecture.mmd": "flowchart TD\n A[Client] --> B[Run]\n"
"recommendations.json": SAMPLE_DATA["recommendations.json"]
},
},
"target_architecture": {
"content": "## Elements\n## Relationships\n## Checks\n",
"attachments": {
"target-architecture.json": GRAPH,
"architecture.mmd": render_mermaid(parse_graph(GRAPH)),
},
},
"deployment_guidance": {
@@ -52,8 +68,13 @@ def execution(status="completed", **overrides) -> Interaction:
"attachments": {
"terraform/main.tf": "resource {}",
"terraform/variables.tf": "",
"implementation.json": SAMPLE_DATA["implementation.json"],
},
},
"validation_results": {
"content": "## Deployment dry-run\nPass.",
"attachments": {"verdicts.json": SAMPLE_DATA["verdicts.json"]},
},
"solution_guide": {"content": guide(), "attachments": {}},
}
for kind, value in overrides.items():
@@ -148,7 +169,7 @@ async def test_invalid_terraform_and_diagram_fail_and_warnings_fail_completion()
result = await score_execution(
CASE,
execution("completed_with_warnings", architecture_diagram=diagram),
execution("completed_with_warnings", target_architecture=diagram),
HEADINGS,
failing_terraform,
)
@@ -181,3 +202,276 @@ def test_benchmark_cases_load_with_valid_answer_patterns():
for pattern, answer in case.scripted_answers:
re.compile(pattern)
assert answer
# --- checks that can see quality (iteration 4, step 8) ---------------------------------
class FakeJudge:
"""Scores every question with a fixed value, or per id from a mapping."""
def __init__(self, default=1.0, **by_id):
self.default, self.by_id = default, by_id
self.asked: list[dict] = []
async def answer(self, document, questions):
self.asked.append(dict(questions))
return {key: self.by_id.get(key, self.default) for key in questions}
CHECKLISTS = {
"reliability": ["No single point of failure.", "Backups are tested."],
"security": ["Least privilege is applied."],
}
def deliverable(content: str, **attachments: str) -> dict:
return {"content": content, "attachments": attachments}
async def test_a_requirement_that_reaches_no_check_breaks_the_chain():
import json
requirements = json.loads(SAMPLE_DATA["requirements.json"])
requirements.append({"id": "NFR-2", "kind": "non_functional", "text": "Retain."})
result = await score_execution(
CASE,
execution(
requirements=deliverable(
"r", **{"requirements.json": json.dumps(requirements)}
)
),
HEADINGS,
passing_terraform,
)
chain = check(result, "requirements_chain_complete")
assert not chain.passed
assert chain.detail == "Requirement NFR-2 has no Check"
async def test_a_check_left_failed_or_unverifiable_fails_the_case():
import json
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
verdicts["checks"][0]["verdict"] = "fail"
verdicts["checks"][1]["verdict"] = "not_verifiable"
verdicts["proposed_additions"] = [
{"statement": "s", "applies_to": "E-1", "reason": "NFR-1"}
]
result = await score_execution(
CASE,
execution(
validation_results=deliverable(
"v", **{"verdicts.json": json.dumps(verdicts)}
)
),
HEADINGS,
passing_terraform,
)
assert (
check(result, "every_check_passes").detail == "K-1 failed; K-2 not verifiable"
)
async def test_a_diagram_that_is_not_the_graphs_fails():
drawn = deliverable(
"t",
**{
"target-architecture.json": GRAPH,
"architecture.mmd": "flowchart LR\n a --> b\n",
},
)
result = await score_execution(
CASE, execution(target_architecture=drawn), HEADINGS, passing_terraform
)
assert not check(result, "diagram_matches_graph").passed
assert check(await _complete(), "diagram_matches_graph").passed
async def _complete():
return await score_execution(CASE, execution(), HEADINGS, passing_terraform)
async def test_judged_checks_run_only_with_a_judge():
without = await score_execution(CASE, execution(), HEADINGS, passing_terraform)
judged = await score_execution(
CASE,
execution(),
HEADINGS,
passing_terraform,
judge=FakeJudge(),
pillar_checklists=CHECKLISTS,
)
names = {c.name for c in judged.checks} - {c.name for c in without.checks}
assert names == {"pillar_checklists_covered", "recommendations_quality"}
assert judged.score == 1.0
async def test_checklist_coverage_is_judged_per_pillar_and_the_weakest_decides():
judge = FakeJudge(**{"reliability.1": 0.0})
result = await score_execution(
CASE,
execution(),
HEADINGS,
passing_terraform,
judge=judge,
pillar_checklists=CHECKLISTS,
)
coverage = check(result, "pillar_checklists_covered")
assert not coverage.passed
assert coverage.detail == "reliability 50%, security 100%"
assert "Backups are tested." in judge.asked[0]["reliability.1"]["text"]
async def test_generic_recommendations_fail_the_quality_rubric():
result = await score_execution(
CASE,
execution(),
HEADINGS,
passing_terraform,
judge=FakeJudge(default=1.0, specific=0.0, actionable=0.33, traceable=0.33),
pillar_checklists=CHECKLISTS,
)
quality = check(result, "recommendations_quality")
assert not quality.passed
assert quality.detail.startswith("mean 0.42")
async def test_an_interactive_case_checks_specialists_ask_before_submitting():
from dataclasses import replace
interactive = replace(CASE, interactive=True)
asked = [
{"deliverable": "requirements", "outcome": "NeedsInput"},
{"deliverable": "requirements", "outcome": "Submitted"},
{"deliverable": None, "outcome": "TurnEnded"},
]
silent = [*asked, {"deliverable": "product_selection", "outcome": "Submitted"}]
good = await score_execution(
interactive, execution(), HEADINGS, passing_terraform, timeline=asked
)
bad = await score_execution(
interactive, execution(), HEADINGS, passing_terraform, timeline=silent
)
batch = await score_execution(CASE, execution(), HEADINGS, passing_terraform)
assert check(good, "specialists_ask_before_submitting").passed
assert (
check(bad, "specialists_ask_before_submitting").detail
== "submitted without asking: product_selection"
)
assert "specialists_ask_before_submitting" not in {c.name for c in batch.checks}
def test_exceeding_a_threshold_is_a_regression():
from dataclasses import replace
from eval.scoring import regressions
case = replace(CASE, thresholds={"revisions": 2, "seconds": 3000, "tokens": 4e6})
assert regressions(case, {"revisions": 2, "seconds": 2999.5, "tokens": 3.9e6}) == []
assert regressions(case, {"revisions": 3, "seconds": 3100, "tokens": None}) == [
"revisions: 3 > 2",
"seconds: 3100 > 3000",
]
assert regressions(CASE, {"revisions": 9}) == []
def test_every_pillar_skill_yields_checklist_items():
from app.providers import load_provider
from eval.scoring import checklist_items
bundle = load_provider("gcp")
for pillar, skill in bundle.pillar_skills.items():
items = checklist_items(bundle.skills[skill].path / "SKILL.md")
assert len(items) >= 3, pillar
assert all(
not item.startswith(("-", "*")) and "\n" not in item for item in items
)
def test_a_judge_reply_is_parsed_defensively():
from eval.judge import parse_scores
questions = {"a": {}, "b": {}, "c": {}, "d": {}}
assert parse_scores('{"a": 1, "b": 0.33, "c": true, "d": 7}', questions) == {
"a": 1.0,
"b": 0.33,
"c": 1.0,
"d": 1.0,
}
assert parse_scores("not json", questions) == dict.fromkeys(questions, 0.0)
assert parse_scores('{"a": "yes"}', questions)["a"] == 0.0
async def test_a_scripted_user_drives_an_interactive_execution(tmp_path):
from dataclasses import replace
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.runtime import RoleSpec
from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
from eval.solution_eval import run_with_scripted_user
requirements = DeliverableKind.REQUIREMENTS
analyst = DEPENDENCY_GRAPH[requirements].specialist
roles = {
"orchestrator": RoleSpec("orchestrator", "", budget=10),
**{
spec.specialist: RoleSpec(spec.specialist, "", budget=10)
for spec in DEPENDENCY_GRAPH.values()
},
}
runtime = FakeAgentRuntime(
{
"orchestrator": [
[Call("delegate", {"kinds": ["requirements"]})],
[Call("ask_user")],
[Call("delegate", {"kinds": ["requirements"]})],
[Call("ask_user", {"questions": ["Which region?"]})],
[Call("ask_user", {"questions": ["Anything else?"]})],
],
analyst: [
[Call("needs_input", {"questions": ["Do you approve this draft?"]})],
[sample_submission(requirements)],
],
}
)
engine = ExecutionEngine(
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
)
case = replace(
CASE,
interactive=True,
scripted_answers=(("approve", "Approved."), ("region", "europe-west2.")),
)
final = await run_with_scripted_user(
engine, case, "exec-interactive", max_user_turns=2
)
replies = [m["text"] for m in final.state["conversation"] if m["from"] == "user"]
assert any("Approved." in reply for reply in replies)
assert any("europe-west2." in reply for reply in replies)
assert final.state["deliverables"]["requirements"]["content"]