feat: establish the Target architecture as a graph before code

Design is now approved before any infrastructure code is written (ADR-0005).
A new target-architect Specialist produces one graph of Elements,
Relationships and Checks; the diagram is rendered from it, deployment
guidance must account for every Check, and validation gives a verdict per
Check. Requirements, components, products and recommendations carry ids in
data files, and a submission with a gap in the Requirements chain is refused.

Also in this commit:
- Customer cases group Executions, with an intent history per case
- lesson candidates are captured when an Execution ends
- evaluation gains chain, verdict, judged and interactive checks, and
  per-case thresholds
- Terraform providers are cached between checks
- the requirements analyst proposes defaults instead of asking
- the legacy-baseline case and the checks that only read it are removed
- ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed
- plans for iterations 4 to 7 under .agents/plans/

Unproven with a real model: no evaluation run has completed on this design.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-07 13:59:09 +01:00
parent 7ee49ab352
commit e9246828b3
586 changed files with 5166 additions and 22432 deletions

View File

@@ -2,10 +2,16 @@
Each check is a small, explainable rule tied to the upstream Skill's requirements.
A case passes when its score reaches PASS_THRESHOLD.
Structural checks always run. Judged checks (pillar checklist coverage, recommendation
quality) run when a judge is given, and are what lets a better or worse Skill move the
score. Thresholds on revisions, seconds and tokens turn a slower or costlier run into a
reported regression without changing the score.
"""
from __future__ import annotations
import json
import re
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass, field
@@ -15,9 +21,43 @@ from typing import Any
from app.adk.runtime import ToolResult
from app.agent import Interaction
from app.tools.local import mermaid_check
from app.workflows.chain import (
COMPONENTS_FILE,
PRODUCTS_FILE,
RECOMMENDATIONS_FILE,
REQUIREMENTS_FILE,
VERDICTS_FILE,
read_graph,
read_list,
validation_outcome,
)
from app.workflows.deliverables import DeliverableKind
from app.workflows.target_architecture import (
DIAGRAM_ATTACHMENT,
GRAPH_ATTACHMENT,
GraphError,
chain_gaps,
parse_graph,
render_mermaid,
)
from eval.judge import Judge
PASS_THRESHOLD = 0.8
CHECKLIST_COVERAGE = 0.7 # share of a pillar Skill's checklist the design must address
QUALITY_BAR = 0.67 # mean of the graded rubric: "mostly" or better
THRESHOLD_METRICS = ("revisions", "seconds", "tokens")
# What a reviewer would ask of design recommendations, graded 0 to 1 each.
QUALITY_RUBRIC: Mapping[str, str] = {
"specific": "Are the recommendations specific to this workload and its stated "
"requirements, as opposed to generic best practice that would fit any workload?",
"actionable": "Does each recommendation name the concrete setting, feature or "
"configuration to apply, so an engineer could act on it without more research?",
"trade_offs": "Where pillars pull in different directions (for example cost "
"against reliability), is the trade-off named and a resolution recommended?",
"traceable": "Can each recommendation be traced to a requirement or constraint "
"of the workload?",
}
TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]]
_HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE)
@@ -32,6 +72,11 @@ class CaseSpec:
workflow_request: str
required_products: tuple[str, ...]
scripted_answers: tuple[tuple[str, str], ...]
# Run with a user in the loop (scripted): Approvals and questions are answered
# turn by turn, and Specialists are checked for asking before they submit.
interactive: bool = False
# Upper bounds for "revisions", "seconds" and "tokens"; exceeding one is a regression.
thresholds: Mapping[str, float] = field(default_factory=dict)
@dataclass(frozen=True)
@@ -62,8 +107,6 @@ class CaseResult:
def load_cases(path: Path) -> list[CaseSpec]:
import json
return [
CaseSpec(
id=item["id"],
@@ -73,6 +116,8 @@ def load_cases(path: Path) -> list[CaseSpec]:
scripted_answers=tuple(
tuple(pair) for pair in item.get("scripted_answers", [])
),
interactive=bool(item.get("interactive", False)),
thresholds=dict(item.get("thresholds", {})),
)
for item in json.loads(path.read_text(encoding="utf-8"))
]
@@ -87,6 +132,53 @@ def template_headings(template: Path) -> list[str]:
]
def checklist_items(skill_file: Path) -> list[str]:
"""The bullet items under a pillar Skill's "Validation checklist" heading."""
text = skill_file.read_text(encoding="utf-8")
match = re.search(
r"^##\s+Validation checklist\s*$(.*?)(?=^##\s|\Z)",
text,
re.MULTILINE | re.DOTALL,
)
if not match:
return []
# Bullets may nest: a bullet with deeper bullets under it is a category, and the
# items to check are the leaves, each prefixed with its category.
bullets: list[tuple[int, str]] = []
for line in match.group(1).splitlines():
bullet = re.match(r"^(\s*)[-*]\s+(.*)$", line)
if bullet:
text = re.sub(r"^\[[ xX]?\]\s*", "", bullet.group(2).strip())
bullets.append((len(bullet.group(1)), text.replace("**", "")))
elif bullets and line.startswith(" ") and line.strip():
indent, text = bullets[-1]
bullets[-1] = (indent, f"{text} {line.strip()}")
items: list[str] = []
category = ""
for index, (indent, text) in enumerate(bullets):
has_children = index + 1 < len(bullets) and bullets[index + 1][0] > indent
if has_children:
category = text.rstrip(":").strip()
elif indent == 0:
category = ""
items.append(text)
else:
items.append(f"{category}: {text}" if category else text)
return items
def regressions(case: CaseSpec, measured: Mapping[str, float | None]) -> list[str]:
"""Each threshold the run exceeded, as "metric: measured > limit"."""
over = []
for metric in THRESHOLD_METRICS:
limit = case.thresholds.get(metric)
value = measured.get(metric)
if limit is not None and value is not None and value > limit:
over.append(f"{metric}: {value:g} > {limit:g}")
return over
def _normalise(text: str) -> str:
return " ".join(_NUMBERING.sub("", text.strip().lower()).split())
@@ -103,6 +195,10 @@ async def score_execution(
interaction: Interaction,
headings: Sequence[str],
terraform_validate: TerraformValidate,
*,
judge: Judge | None = None,
pillar_checklists: Mapping[str, Sequence[str]] | None = None,
timeline: Sequence[Mapping[str, Any]] = (),
) -> CaseResult:
deliverables: Mapping[str, Mapping[str, Any]] = interaction.state.get(
"deliverables", {}
@@ -179,7 +275,7 @@ async def score_execution(
)
)
diagram = attachments(DeliverableKind.ARCHITECTURE_DIAGRAM).get(
diagram = attachments(DeliverableKind.TARGET_ARCHITECTURE).get(
"architecture.mmd", ""
)
mermaid = await mermaid_check(diagram)
@@ -204,4 +300,129 @@ async def score_execution(
terraform_ok, terraform_detail = False, "no Terraform files attached"
checks.append(Check("terraform_validates", terraform_ok, terraform_detail))
checks.append(_chain_check(deliverables))
checks.append(_verdicts_check(attachments(DeliverableKind.VALIDATION_RESULTS)))
checks.append(
_diagram_matches_graph(attachments(DeliverableKind.TARGET_ARCHITECTURE))
)
if case.interactive:
checks.append(_asks_before_submitting(timeline))
if judge is not None:
recommendations = content(DeliverableKind.DESIGN_RECOMMENDATIONS)
checks.append(
await _checklist_coverage(judge, recommendations, pillar_checklists or {})
)
checks.append(await _quality(judge, recommendations))
return CaseResult(case.id, interaction.status, tuple(checks))
def _chain_check(deliverables: Mapping[str, Mapping[str, Any]]) -> Check:
"""The Requirements chain holds end to end: every requirement reaches a Check,
directly or through a component's Element, and nothing along it is missing."""
name = "requirements_chain_complete"
graph = read_graph(deliverables)
if graph is None:
return Check(name, False, "no well-formed Target architecture")
gaps = chain_gaps(
graph,
requirements=read_list(deliverables, "requirements", REQUIREMENTS_FILE),
components=read_list(deliverables, "technical_decomposition", COMPONENTS_FILE),
products=read_list(deliverables, "product_selection", PRODUCTS_FILE),
recommendations=read_list(
deliverables, "design_recommendations", RECOMMENDATIONS_FILE
),
)
return Check(name, not gaps, "; ".join(gaps))
def _verdicts_check(attachments: Mapping[str, str]) -> Check:
"""Validation ended with every Check passed and the dry-run not failed."""
name = "every_check_passes"
text = attachments.get(VERDICTS_FILE)
if text is None:
return Check(name, False, f"no {VERDICTS_FILE}")
outcome = validation_outcome(text)
open_checks = [f"{item['check']} failed" for item in outcome["failed"]] + [
f"{item['check']} not verifiable" for item in outcome["not_verifiable"]
]
if outcome["dry_run"].get("verdict") == "fail":
open_checks.append("dry-run failed")
if not outcome["passed"] and not open_checks:
open_checks.append("no verdicts")
return Check(name, not open_checks, "; ".join(open_checks))
def _diagram_matches_graph(attachments: Mapping[str, str]) -> Check:
name = "diagram_matches_graph"
try:
graph = parse_graph(attachments.get(GRAPH_ATTACHMENT, ""))
except GraphError as exc:
return Check(name, False, exc.problems[0])
matches = attachments.get(DIAGRAM_ATTACHMENT) == render_mermaid(graph)
return Check(name, matches, "" if matches else "the diagram is not the graph's")
def _asks_before_submitting(timeline: Sequence[Mapping[str, Any]]) -> Check:
"""In an interactive Execution every Specialist confirms with the user before its
first submission (spec risk D32)."""
asked: set[str] = set()
silent: list[str] = []
seen: set[str] = set()
for run in timeline:
deliverable = run.get("deliverable")
if not deliverable:
continue
if run.get("outcome") == "NeedsInput":
asked.add(deliverable)
elif run.get("outcome") == "Submitted" and deliverable not in seen:
seen.add(deliverable)
if deliverable not in asked:
silent.append(deliverable)
return Check(
"specialists_ask_before_submitting",
bool(seen) and not silent,
"submitted without asking: " + ", ".join(silent) if silent else "",
)
async def _checklist_coverage(
judge: Judge, recommendations: str, checklists: Mapping[str, Sequence[str]]
) -> Check:
name = "pillar_checklists_covered"
questions = {
f"{pillar}.{index}": {
"kind": "yes_no",
"text": f"Do the recommendations address this {pillar} checklist item, "
f"either by recommending it or by explaining why it does not apply? {item}",
}
for pillar, items in checklists.items()
for index, item in enumerate(items)
}
if not recommendations or not questions:
return Check(name, False, "no design recommendations or no checklists")
scores = await judge.answer(recommendations, questions)
per_pillar = {
pillar: sum(scores.get(f"{pillar}.{i}", 0) for i in range(len(items)))
/ len(items)
for pillar, items in checklists.items()
if items
}
weakest = min(per_pillar.values())
detail = ", ".join(f"{pillar} {share:.0%}" for pillar, share in per_pillar.items())
return Check(name, weakest >= CHECKLIST_COVERAGE, detail)
async def _quality(judge: Judge, recommendations: str) -> Check:
name = "recommendations_quality"
if not recommendations:
return Check(name, False, "no design recommendations")
scores = await judge.answer(
recommendations,
{key: {"kind": "graded", "text": text} for key, text in QUALITY_RUBRIC.items()},
)
mean = sum(scores.values()) / len(QUALITY_RUBRIC)
detail = ", ".join(f"{key} {scores.get(key, 0):.2f}" for key in QUALITY_RUBRIC)
return Check(name, mean >= QUALITY_BAR, f"mean {mean:.2f}: {detail}")