Files
gcp_solution_architecture_a…/eval/scoring.py
Jonathan Boniface ef37f59bad feat: opt-in published Terraform modules from approved sources (iteration 8)
Infrastructure code is plain resources by default. An Execution may opt in
with terraform_modules; the deployment default is TERRAFORM_MODULES, off.

- A module is accepted only from the registry namespaces the provider bundle
  lists (terraform-google-modules), at one exact version. Any other source is
  refused before Terraform runs; until now nothing stopped generated code
  naming any source for terraform init to download.
- terraform_module looks up a module's inputs, outputs and submodules from
  the registry and says when it cannot be used with the pinned provider.
- skills/google-cloud-iac-modules: when a module fits, and four modules with
  examples a test validates against hashicorp/google 8.6.0.
- Opted in, the IaC author and validator get the Skill and the lookup. Not
  opted in, they get neither and a module block is refused in code.
- A Change request follows its Baseline. execution.json records the choice.
- case-07 runs case 03's request with modules opted into.

Azure (aztfm) and AWS (terraform-aws-modules) sources are recorded in the
plan for when those Cloud providers are added. No case has run with this.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 22:09:29 +01:00

582 lines
21 KiB
Python

"""Score a finished Execution against a benchmark case.
Each check is a small, explainable rule tied to the upstream Skill's requirements.
A case passes when its score reaches PASS_THRESHOLD.
Structural checks always run. Judged checks (pillar checklist coverage, recommendation
quality) run when a judge is given, and are what lets a better or worse Skill move the
score. Thresholds on revisions, seconds and tokens turn a slower or costlier run into a
reported regression without changing the score.
"""
from __future__ import annotations
import json
import re
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from app.adk.runtime import ToolResult
from app.agent import Interaction
from app.tools.assessment import QUALITY_BAR, QUALITY_RUBRIC
from app.tools.local import mermaid_check
from app.workflows.chain import (
COMPONENTS_FILE,
PRODUCTS_FILE,
RECOMMENDATIONS_FILE,
REQUIREMENTS_FILE,
VERDICTS_FILE,
read_graph,
read_list,
validation_outcome,
)
from app.workflows.deliverables import DeliverableKind
from app.workflows.target_architecture import (
DIAGRAM_ATTACHMENT,
GRAPH_ATTACHMENT,
GraphError,
chain_gaps,
parse_graph,
render_mermaid,
)
from eval.judge import Judge, JudgeUnavailable
PASS_THRESHOLD = 0.8
CHECKLIST_COVERAGE = 0.7 # share of a pillar Skill's checklist the design must address
THRESHOLD_METRICS = ("revisions", "seconds", "tokens")
TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]]
_HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE)
_NUMBERING = re.compile(r"^\d+(?:\.\d+)*\.?\s+")
_DOC_LINK = re.compile(r"https://(?:docs\.)?cloud\.google\.com/\S+")
@dataclass(frozen=True)
class CaseSpec:
id: str
name: str
workflow_request: str
required_products: tuple[str, ...]
scripted_answers: tuple[tuple[str, ...], ...] # (pattern, answer[, asker])
# Run with a user in the loop (scripted): Approvals and questions are answered
# turn by turn, and Specialists are checked for asking before they submit.
interactive: bool = False
# Upper bounds for "revisions", "seconds" and "tokens"; exceeding one is a regression.
thresholds: Mapping[str, float] = field(default_factory=dict)
# A change case: the request is run to completion, then this statement of what
# has changed is run as a Change request against it (ADR-0006).
change: str = ""
# Overlay Skills the case's Executions name (ADR-0004).
overlay_skills: tuple[str, ...] = ()
# Whether the case's Executions opt into published Terraform modules.
terraform_modules: bool = False
# What the infrastructure code must and must not contain, as regular
# expressions over all of it: how a case checks that standards from an overlay
# Skill reached the code. Each is {"name", "require"} or {"name", "forbid"}.
code_standards: tuple[Mapping[str, str], ...] = ()
@dataclass(frozen=True)
class Check:
name: str
passed: bool
detail: str = ""
@dataclass(frozen=True)
class CaseResult:
case_id: str
status: str
checks: tuple[Check, ...]
details: Mapping[str, Any] = field(default_factory=dict)
@property
def score(self) -> float:
return (
sum(c.passed for c in self.checks) / len(self.checks)
if self.checks
else 0.0
)
@property
def passed(self) -> bool:
return self.score >= PASS_THRESHOLD
def load_cases(path: Path) -> list[CaseSpec]:
return [
CaseSpec(
id=item["id"],
name=item["name"],
workflow_request=item["workflow_request"],
required_products=tuple(item["required_products"]),
scripted_answers=tuple(
tuple(pair) for pair in item.get("scripted_answers", [])
),
interactive=bool(item.get("interactive", False)),
thresholds=dict(item.get("thresholds", {})),
change=str(item.get("change", "")),
overlay_skills=tuple(item.get("overlay_skills", ())),
terraform_modules=bool(item.get("terraform_modules", False)),
code_standards=tuple(item.get("code_standards", ())),
)
for item in json.loads(path.read_text(encoding="utf-8"))
]
def template_headings(template: Path) -> list[str]:
"""Section headings (level 2+) of a solution-guide template, placeholders removed."""
return [
heading
for _, heading in _HEADING.findall(template.read_text(encoding="utf-8"))
if "[" not in heading
]
def checklist_items(skill_file: Path) -> list[str]:
"""The bullet items under a pillar Skill's "Validation checklist" heading."""
text = skill_file.read_text(encoding="utf-8")
match = re.search(
r"^##\s+Validation checklist\s*$(.*?)(?=^##\s|\Z)",
text,
re.MULTILINE | re.DOTALL,
)
if not match:
return []
# Bullets may nest: a bullet with deeper bullets under it is a category, and the
# items to check are the leaves, each prefixed with its category.
bullets: list[tuple[int, str]] = []
for line in match.group(1).splitlines():
bullet = re.match(r"^(\s*)[-*]\s+(.*)$", line)
if bullet:
text = re.sub(r"^\[[ xX]?\]\s*", "", bullet.group(2).strip())
bullets.append((len(bullet.group(1)), text.replace("**", "")))
elif bullets and line.startswith(" ") and line.strip():
indent, text = bullets[-1]
bullets[-1] = (indent, f"{text} {line.strip()}")
items: list[str] = []
category = ""
for index, (indent, text) in enumerate(bullets):
has_children = index + 1 < len(bullets) and bullets[index + 1][0] > indent
if has_children:
category = text.rstrip(":").strip()
elif indent == 0:
category = ""
items.append(text)
else:
items.append(f"{category}: {text}" if category else text)
return items
def regressions(case: CaseSpec, measured: Mapping[str, float | None]) -> list[str]:
"""Each threshold the run exceeded, as "metric: measured > limit"."""
over = []
for metric in THRESHOLD_METRICS:
limit = case.thresholds.get(metric)
value = measured.get(metric)
if limit is not None and value is not None and value > limit:
over.append(f"{metric}: {value:g} > {limit:g}")
return over
def _normalise(text: str) -> str:
return " ".join(_NUMBERING.sub("", text.strip().lower()).split())
def _mentions(text: str, product: str) -> bool:
return (
re.search(rf"(?<![\w/]){re.escape(product.lower())}(?![\w/])", text.lower())
is not None
)
async def score_execution(
case: CaseSpec,
interaction: Interaction,
headings: Sequence[str],
terraform_validate: TerraformValidate,
*,
judge: Judge | None = None,
pillar_checklists: Mapping[str, Sequence[str]] | None = None,
timeline: Sequence[Mapping[str, Any]] = (),
) -> CaseResult:
deliverables: Mapping[str, Mapping[str, Any]] = interaction.state.get(
"deliverables", {}
)
def content(kind: DeliverableKind) -> str:
item = deliverables.get(kind.value)
return item["content"] if item else ""
def attachments(kind: DeliverableKind) -> Mapping[str, str]:
item = deliverables.get(kind.value)
return item.get("attachments", {}) if item else {}
checks: list[Check] = []
checks.append(
Check(
"completed",
interaction.status == "completed",
"; ".join(interaction.state.get("warnings", [])) or interaction.status,
)
)
guide = content(DeliverableKind.SOLUTION_GUIDE)
guide_headings = {_normalise(h) for _, h in _HEADING.findall(guide)}
missing_sections = [h for h in headings if _normalise(h) not in guide_headings]
checks.append(
Check(
"guide_follows_template",
bool(guide) and not missing_sections,
"missing: " + "; ".join(missing_sections) if missing_sections else "",
)
)
selection = content(DeliverableKind.PRODUCT_SELECTION)
selected_text = selection + "\n" + guide
missing_products = [
p for p in case.required_products if not _mentions(selected_text, p)
]
checks.append(
Check(
"required_products_selected",
bool(selection) and not missing_products,
"missing: " + ", ".join(missing_products) if missing_products else "",
)
)
lowered = selection.lower()
checks.append(
Check(
"selection_has_alternatives",
"alternative" in lowered and "pros" in lowered and "cons" in lowered,
)
)
checks.append(
Check(
"selection_cites_documentation",
bool(_DOC_LINK.search(selection)),
)
)
requirements = content(DeliverableKind.REQUIREMENTS)
leaked = [
product
for product in case.required_products
if _mentions(requirements, product)
and not _mentions(case.workflow_request, product)
]
checks.append(
Check(
"requirements_name_no_products",
bool(requirements) and not leaked,
"names: " + ", ".join(leaked) if leaked else "",
)
)
diagram = attachments(DeliverableKind.TARGET_ARCHITECTURE).get(
"architecture.mmd", ""
)
mermaid = await mermaid_check(diagram)
checks.append(
Check(
"diagram_is_valid_mermaid",
bool(diagram) and mermaid.message.startswith("Mermaid check passed"),
mermaid.message,
)
)
terraform_files = {
name.removeprefix("terraform/"): text
for name, text in attachments(DeliverableKind.DEPLOYMENT_GUIDANCE).items()
if name.endswith((".tf", ".tfvars", ".tf.json"))
}
if terraform_files:
validation = await terraform_validate(terraform_files)
terraform_ok = validation.message.startswith("terraform validate passed")
terraform_detail = validation.message[:1500]
else:
terraform_ok, terraform_detail = False, "no Terraform files attached"
checks.append(Check("terraform_validates", terraform_ok, terraform_detail))
if case.code_standards:
checks.append(_code_standards(case.code_standards, terraform_files))
checks.append(_chain_check(deliverables))
checks.append(_verdicts_check(attachments(DeliverableKind.VALIDATION_RESULTS)))
checks.append(
_diagram_matches_graph(attachments(DeliverableKind.TARGET_ARCHITECTURE))
)
if case.interactive:
checks.append(_asks_before_submitting(timeline))
if judge is not None:
recommendations = content(DeliverableKind.DESIGN_RECOMMENDATIONS)
checks.append(
await _checklist_coverage(judge, recommendations, pillar_checklists or {})
)
checks.append(await _quality(judge, recommendations))
return CaseResult(case.id, interaction.status, tuple(checks))
def _code_standards(
standards: Sequence[Mapping[str, str]], files: Mapping[str, str]
) -> Check:
"""Whether the code holds what the case requires of it and nothing it forbids."""
code = "\n".join(files.values())
broken = []
for standard in standards:
required, forbidden = standard.get("require"), standard.get("forbid")
if required and not re.search(required, code):
broken.append(f"{standard['name']}: not found")
found = re.search(forbidden, code) if forbidden else None
if found:
broken.append(f"{standard['name']}: found {found.group(0)}")
return Check("code_follows_standards", bool(code) and not broken, "; ".join(broken))
def _chain_check(deliverables: Mapping[str, Mapping[str, Any]]) -> Check:
"""The Requirements chain holds end to end: every requirement reaches a Check,
directly or through a component's Element, and nothing along it is missing."""
name = "requirements_chain_complete"
graph = read_graph(deliverables)
if graph is None:
return Check(name, False, "no well-formed Target architecture")
gaps = chain_gaps(
graph,
requirements=read_list(deliverables, "requirements", REQUIREMENTS_FILE),
components=read_list(deliverables, "technical_decomposition", COMPONENTS_FILE),
products=read_list(deliverables, "product_selection", PRODUCTS_FILE),
recommendations=read_list(
deliverables, "design_recommendations", RECOMMENDATIONS_FILE
),
)
return Check(name, not gaps, "; ".join(gaps))
def _verdicts_check(attachments: Mapping[str, str]) -> Check:
"""Validation ended with every Check passed and the dry-run not failed."""
name = "every_check_passes"
text = attachments.get(VERDICTS_FILE)
if text is None:
return Check(name, False, f"no {VERDICTS_FILE}")
outcome = validation_outcome(text)
open_checks = [f"{item['check']} failed" for item in outcome["failed"]] + [
f"{item['check']} not verifiable" for item in outcome["not_verifiable"]
]
if outcome["dry_run"].get("verdict") == "fail":
open_checks.append("dry-run failed")
if not outcome["passed"] and not open_checks:
open_checks.append("no verdicts")
return Check(name, not open_checks, "; ".join(open_checks))
def _diagram_matches_graph(attachments: Mapping[str, str]) -> Check:
name = "diagram_matches_graph"
try:
graph = parse_graph(attachments.get(GRAPH_ATTACHMENT, ""))
except GraphError as exc:
return Check(name, False, exc.problems[0])
matches = attachments.get(DIAGRAM_ATTACHMENT) == render_mermaid(graph)
return Check(name, matches, "" if matches else "the diagram is not the graph's")
def _asks_before_submitting(timeline: Sequence[Mapping[str, Any]]) -> Check:
"""In an interactive Execution every Specialist confirms with the user before its
first submission (spec risk D32)."""
asked: set[str] = set()
silent: list[str] = []
seen: set[str] = set()
for run in timeline:
deliverable = run.get("deliverable")
if not deliverable:
continue
if run.get("outcome") == "NeedsInput":
asked.add(deliverable)
elif run.get("outcome") == "Submitted" and deliverable not in seen:
seen.add(deliverable)
if deliverable not in asked:
silent.append(deliverable)
return Check(
"specialists_ask_before_submitting",
bool(seen) and not silent,
"submitted without asking: " + ", ".join(silent) if silent else "",
)
async def _checklist_coverage(
judge: Judge, recommendations: str, checklists: Mapping[str, Sequence[str]]
) -> Check:
name = "pillar_checklists_covered"
questions = {
f"{pillar}.{index}": {
"kind": "yes_no",
"text": f"Do the recommendations address this {pillar} checklist item, "
f"either by recommending it or by explaining why it does not apply? {item}",
}
for pillar, items in checklists.items()
for index, item in enumerate(items)
}
if not recommendations or not questions:
return Check(name, False, "no design recommendations or no checklists")
try:
scores = await judge.answer(recommendations, questions)
except JudgeUnavailable as exc:
return Check(name, False, f"judge unavailable: {exc}")
per_pillar = {
pillar: sum(scores.get(f"{pillar}.{i}", 0) for i in range(len(items)))
/ len(items)
for pillar, items in checklists.items()
if items
}
weakest = min(per_pillar.values())
detail = ", ".join(f"{pillar} {share:.0%}" for pillar, share in per_pillar.items())
return Check(name, weakest >= CHECKLIST_COVERAGE, detail)
async def _quality(judge: Judge, recommendations: str) -> Check:
name = "recommendations_quality"
if not recommendations:
return Check(name, False, "no design recommendations")
try:
scores = await judge.answer(
recommendations,
{
key: {"kind": "graded", "text": text}
for key, text in QUALITY_RUBRIC.items()
},
)
except JudgeUnavailable as exc:
return Check(name, False, f"judge unavailable: {exc}")
mean = sum(scores.values()) / len(QUALITY_RUBRIC)
detail = ", ".join(f"{key} {scores.get(key, 0):.2f}" for key in QUALITY_RUBRIC)
return Check(name, mean >= QUALITY_BAR, f"mean {mean:.2f}: {detail}")
# --- a Change request stays contained (iteration 7, step 7) ------------------------------
UNREQUESTED_SHARE = 0.1 # of the Baseline graph's items, changed without being reached
def score_change(
interaction: Interaction, baseline_tokens: float | None, tokens: float | None
) -> list[Check]:
"""Checks on a Change request against its Baseline.
They read what the Execution recorded: the requirements' own account of what
changed, and a comparison by id of the two versions' graphs.
"""
from app.workflows.changes import ADDED, CHANGED, REMOVED, change_summary
state = interaction.state
baseline = (state.get("baseline") or {}).get("deliverables") or {}
deliverables = state.get("deliverables", {})
summary = change_summary(baseline, deliverables)
requirements = summary["requirements"]
checks = [
Check(
"change_is_recorded",
bool(baseline) and bool(requirements),
", ".join(f"{item['id']} {item['change']}" for item in requirements)
or "no requirement is marked added, changed or removed",
)
]
graph = _graph_data(deliverables)
touched = set(summary["checks"][ADDED]) | set(summary["checks"][CHANGED])
removed_checks = set(summary["checks"][REMOVED])
old_sources = {
item.get("id"): item.get("source") for item in _graph_data(baseline)["checks"]
}
serving: dict[str, set[str]] = {}
for component in read_list(
deliverables, "technical_decomposition", COMPONENTS_FILE
):
for requirement in component.get("requirements") or ():
serving.setdefault(str(requirement), set()).add(str(component.get("id")))
unreached = []
for item in requirements:
identifier = item["id"]
if item["change"] == REMOVED:
# What it asked for should go: a Check that came from it is removed.
reaches = any(old_sources.get(c) == identifier for c in removed_checks)
reaches = reaches or identifier not in old_sources.values()
else:
# A constraint is the source of its Checks. A functional requirement is
# carried by the components that serve it, so it is reached through a
# Check on an Element that realises one of them.
realising = {
element.get("id")
for element in graph["elements"]
if element.get("component") in serving.get(identifier, ())
}
reaches = any(
check.get("id") in touched
and (
check.get("source") == identifier
or check.get("applies_to") in realising
)
for check in graph["checks"]
)
if not reaches:
unreached.append(identifier)
checks.append(
Check(
"changed_requirements_reach_checks",
bool(requirements) and not unreached,
"no Check changed for: " + ", ".join(unreached) if unreached else "",
)
)
unrequested = [
f"{part} {identifier}"
for part, ids in summary["unrequested"].items()
for identifier in ids
]
size = sum(len(items) for items in _graph_data(baseline).values()) or 1
checks.append(
Check(
"change_is_contained",
len(unrequested) <= max(2, UNREQUESTED_SHARE * size),
f"{len(unrequested)} of {size} items changed without being reached"
+ (": " + ", ".join(unrequested[:12]) if unrequested else ""),
)
)
cheaper = (
baseline_tokens is not None and tokens is not None and tokens < baseline_tokens
)
checks.append(
Check(
"change_costs_less_than_baseline",
cheaper,
f"{tokens or 0:,.0f} tokens against {baseline_tokens or 0:,.0f}",
)
)
return checks
def _graph_data(
deliverables: Mapping[str, Mapping[str, Any]],
) -> dict[str, list[dict[str, Any]]]:
text = (
(deliverables.get(DeliverableKind.TARGET_ARCHITECTURE.value) or {})
.get("attachments", {})
.get("target-architecture.json", "")
)
try:
data = json.loads(text)
except ValueError:
data = {}
if not isinstance(data, dict):
data = {}
return {
part: [item for item in data.get(part, []) if isinstance(item, dict)]
for part in ("elements", "relationships", "checks")
}