Files
gcp_solution_architecture_a…/eval/scoring.py
Jonathan Boniface a48468d477
Some checks failed
validation / verify (push) Failing after 11s
feat: what a request does not say is a question for the customer
A pillar's checklist item the request gives no information about was marked
"not addressed", which reads as a fault in the design. It is missing
information. The checklist review gains a fourth outcome, to confirm with the
customer, which must carry the question to ask. The questions are written
into the design recommendations and the guide, and to customer-questions.md;
the evaluation's judged check counts such an item as accounted for.

Records the owner's decisions of 2026-10-10 and the vendor's answers on Jev's
retention, training, context and rate limits (iteration 5 step 1), and the
owner's labels, which showed the judged question was wrong, not the judges.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-10 19:30:12 +01:00

581 lines
21 KiB
Python

"""Score a finished Execution against a benchmark case.
Each check is a small, explainable rule tied to the upstream Skill's requirements.
A case passes when its score reaches PASS_THRESHOLD.
Structural checks always run. Judged checks (pillar checklist coverage, recommendation
quality) run when a judge is given, and are what lets a better or worse Skill move the
score. Thresholds on revisions, seconds and tokens turn a slower or costlier run into a
reported regression without changing the score.
"""
from __future__ import annotations
import json
import re
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from app.adk.runtime import ToolResult
from app.agent import Interaction
from app.tools.assessment import QUALITY_BAR, QUALITY_RUBRIC, checklist_question
from app.tools.local import mermaid_check
from app.workflows.chain import (
COMPONENTS_FILE,
PRODUCTS_FILE,
RECOMMENDATIONS_FILE,
REQUIREMENTS_FILE,
VERDICTS_FILE,
read_graph,
read_list,
validation_outcome,
)
from app.workflows.deliverables import DeliverableKind
from app.workflows.target_architecture import (
DIAGRAM_ATTACHMENT,
GRAPH_ATTACHMENT,
GraphError,
chain_gaps,
parse_graph,
render_mermaid,
)
from eval.judge import Judge, JudgeUnavailable
PASS_THRESHOLD = 0.8
CHECKLIST_COVERAGE = 0.7 # share of a pillar Skill's checklist the design must address
THRESHOLD_METRICS = ("revisions", "seconds", "tokens")
TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]]
_HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE)
_NUMBERING = re.compile(r"^\d+(?:\.\d+)*\.?\s+")
_DOC_LINK = re.compile(r"https://(?:docs\.)?cloud\.google\.com/\S+")
@dataclass(frozen=True)
class CaseSpec:
id: str
name: str
workflow_request: str
required_products: tuple[str, ...]
scripted_answers: tuple[tuple[str, ...], ...] # (pattern, answer[, asker])
# Run with a user in the loop (scripted): Approvals and questions are answered
# turn by turn, and Specialists are checked for asking before they submit.
interactive: bool = False
# Upper bounds for "revisions", "seconds" and "tokens"; exceeding one is a regression.
thresholds: Mapping[str, float] = field(default_factory=dict)
# A change case: the request is run to completion, then this statement of what
# has changed is run as a Change request against it (ADR-0006).
change: str = ""
# Overlay Skills the case's Executions name (ADR-0004).
overlay_skills: tuple[str, ...] = ()
# Whether the case's Executions opt into published Terraform modules.
terraform_modules: bool = False
# What the infrastructure code must and must not contain, as regular
# expressions over all of it: how a case checks that standards from an overlay
# Skill reached the code. Each is {"name", "require"} or {"name", "forbid"}.
code_standards: tuple[Mapping[str, str], ...] = ()
@dataclass(frozen=True)
class Check:
name: str
passed: bool
detail: str = ""
@dataclass(frozen=True)
class CaseResult:
case_id: str
status: str
checks: tuple[Check, ...]
details: Mapping[str, Any] = field(default_factory=dict)
@property
def score(self) -> float:
return (
sum(c.passed for c in self.checks) / len(self.checks)
if self.checks
else 0.0
)
@property
def passed(self) -> bool:
return self.score >= PASS_THRESHOLD
def load_cases(path: Path) -> list[CaseSpec]:
return [
CaseSpec(
id=item["id"],
name=item["name"],
workflow_request=item["workflow_request"],
required_products=tuple(item["required_products"]),
scripted_answers=tuple(
tuple(pair) for pair in item.get("scripted_answers", [])
),
interactive=bool(item.get("interactive", False)),
thresholds=dict(item.get("thresholds", {})),
change=str(item.get("change", "")),
overlay_skills=tuple(item.get("overlay_skills", ())),
terraform_modules=bool(item.get("terraform_modules", False)),
code_standards=tuple(item.get("code_standards", ())),
)
for item in json.loads(path.read_text(encoding="utf-8"))
]
def template_headings(template: Path) -> list[str]:
"""Section headings (level 2+) of a solution-guide template, placeholders removed."""
return [
heading
for _, heading in _HEADING.findall(template.read_text(encoding="utf-8"))
if "[" not in heading
]
def checklist_items(skill_file: Path) -> list[str]:
"""The bullet items under a pillar Skill's "Validation checklist" heading."""
text = skill_file.read_text(encoding="utf-8")
match = re.search(
r"^##\s+Validation checklist\s*$(.*?)(?=^##\s|\Z)",
text,
re.MULTILINE | re.DOTALL,
)
if not match:
return []
# Bullets may nest: a bullet with deeper bullets under it is a category, and the
# items to check are the leaves, each prefixed with its category.
bullets: list[tuple[int, str]] = []
for line in match.group(1).splitlines():
bullet = re.match(r"^(\s*)[-*]\s+(.*)$", line)
if bullet:
text = re.sub(r"^\[[ xX]?\]\s*", "", bullet.group(2).strip())
bullets.append((len(bullet.group(1)), text.replace("**", "")))
elif bullets and line.startswith(" ") and line.strip():
indent, text = bullets[-1]
bullets[-1] = (indent, f"{text} {line.strip()}")
items: list[str] = []
category = ""
for index, (indent, text) in enumerate(bullets):
has_children = index + 1 < len(bullets) and bullets[index + 1][0] > indent
if has_children:
category = text.rstrip(":").strip()
elif indent == 0:
category = ""
items.append(text)
else:
items.append(f"{category}: {text}" if category else text)
return items
def regressions(case: CaseSpec, measured: Mapping[str, float | None]) -> list[str]:
"""Each threshold the run exceeded, as "metric: measured > limit"."""
over = []
for metric in THRESHOLD_METRICS:
limit = case.thresholds.get(metric)
value = measured.get(metric)
if limit is not None and value is not None and value > limit:
over.append(f"{metric}: {value:g} > {limit:g}")
return over
def _normalise(text: str) -> str:
return " ".join(_NUMBERING.sub("", text.strip().lower()).split())
def _mentions(text: str, product: str) -> bool:
return (
re.search(rf"(?<![\w/]){re.escape(product.lower())}(?![\w/])", text.lower())
is not None
)
async def score_execution(
case: CaseSpec,
interaction: Interaction,
headings: Sequence[str],
terraform_validate: TerraformValidate,
*,
judge: Judge | None = None,
pillar_checklists: Mapping[str, Sequence[str]] | None = None,
timeline: Sequence[Mapping[str, Any]] = (),
) -> CaseResult:
deliverables: Mapping[str, Mapping[str, Any]] = interaction.state.get(
"deliverables", {}
)
def content(kind: DeliverableKind) -> str:
item = deliverables.get(kind.value)
return item["content"] if item else ""
def attachments(kind: DeliverableKind) -> Mapping[str, str]:
item = deliverables.get(kind.value)
return item.get("attachments", {}) if item else {}
checks: list[Check] = []
checks.append(
Check(
"completed",
interaction.status == "completed",
"; ".join(interaction.state.get("warnings", [])) or interaction.status,
)
)
guide = content(DeliverableKind.SOLUTION_GUIDE)
guide_headings = {_normalise(h) for _, h in _HEADING.findall(guide)}
missing_sections = [h for h in headings if _normalise(h) not in guide_headings]
checks.append(
Check(
"guide_follows_template",
bool(guide) and not missing_sections,
"missing: " + "; ".join(missing_sections) if missing_sections else "",
)
)
selection = content(DeliverableKind.PRODUCT_SELECTION)
selected_text = selection + "\n" + guide
missing_products = [
p for p in case.required_products if not _mentions(selected_text, p)
]
checks.append(
Check(
"required_products_selected",
bool(selection) and not missing_products,
"missing: " + ", ".join(missing_products) if missing_products else "",
)
)
lowered = selection.lower()
checks.append(
Check(
"selection_has_alternatives",
"alternative" in lowered and "pros" in lowered and "cons" in lowered,
)
)
checks.append(
Check(
"selection_cites_documentation",
bool(_DOC_LINK.search(selection)),
)
)
requirements = content(DeliverableKind.REQUIREMENTS)
leaked = [
product
for product in case.required_products
if _mentions(requirements, product)
and not _mentions(case.workflow_request, product)
]
checks.append(
Check(
"requirements_name_no_products",
bool(requirements) and not leaked,
"names: " + ", ".join(leaked) if leaked else "",
)
)
diagram = attachments(DeliverableKind.TARGET_ARCHITECTURE).get(
"architecture.mmd", ""
)
mermaid = await mermaid_check(diagram)
checks.append(
Check(
"diagram_is_valid_mermaid",
bool(diagram) and mermaid.message.startswith("Mermaid check passed"),
mermaid.message,
)
)
terraform_files = {
name.removeprefix("terraform/"): text
for name, text in attachments(DeliverableKind.DEPLOYMENT_GUIDANCE).items()
if name.endswith((".tf", ".tfvars", ".tf.json"))
}
if terraform_files:
validation = await terraform_validate(terraform_files)
terraform_ok = validation.message.startswith("terraform validate passed")
terraform_detail = validation.message[:1500]
else:
terraform_ok, terraform_detail = False, "no Terraform files attached"
checks.append(Check("terraform_validates", terraform_ok, terraform_detail))
if case.code_standards:
checks.append(_code_standards(case.code_standards, terraform_files))
checks.append(_chain_check(deliverables))
checks.append(_verdicts_check(attachments(DeliverableKind.VALIDATION_RESULTS)))
checks.append(
_diagram_matches_graph(attachments(DeliverableKind.TARGET_ARCHITECTURE))
)
if case.interactive:
checks.append(_asks_before_submitting(timeline))
if judge is not None:
recommendations = content(DeliverableKind.DESIGN_RECOMMENDATIONS)
checks.append(
await _checklist_coverage(judge, recommendations, pillar_checklists or {})
)
checks.append(await _quality(judge, recommendations))
return CaseResult(case.id, interaction.status, tuple(checks))
def _code_standards(
standards: Sequence[Mapping[str, str]], files: Mapping[str, str]
) -> Check:
"""Whether the code holds what the case requires of it and nothing it forbids."""
code = "\n".join(files.values())
broken = []
for standard in standards:
required, forbidden = standard.get("require"), standard.get("forbid")
if required and not re.search(required, code):
broken.append(f"{standard['name']}: not found")
found = re.search(forbidden, code) if forbidden else None
if found:
broken.append(f"{standard['name']}: found {found.group(0)}")
return Check("code_follows_standards", bool(code) and not broken, "; ".join(broken))
def _chain_check(deliverables: Mapping[str, Mapping[str, Any]]) -> Check:
"""The Requirements chain holds end to end: every requirement reaches a Check,
directly or through a component's Element, and nothing along it is missing."""
name = "requirements_chain_complete"
graph = read_graph(deliverables)
if graph is None:
return Check(name, False, "no well-formed Target architecture")
gaps = chain_gaps(
graph,
requirements=read_list(deliverables, "requirements", REQUIREMENTS_FILE),
components=read_list(deliverables, "technical_decomposition", COMPONENTS_FILE),
products=read_list(deliverables, "product_selection", PRODUCTS_FILE),
recommendations=read_list(
deliverables, "design_recommendations", RECOMMENDATIONS_FILE
),
)
return Check(name, not gaps, "; ".join(gaps))
def _verdicts_check(attachments: Mapping[str, str]) -> Check:
"""Validation ended with every Check passed and the dry-run not failed."""
name = "every_check_passes"
text = attachments.get(VERDICTS_FILE)
if text is None:
return Check(name, False, f"no {VERDICTS_FILE}")
outcome = validation_outcome(text)
open_checks = [f"{item['check']} failed" for item in outcome["failed"]] + [
f"{item['check']} not verifiable" for item in outcome["not_verifiable"]
]
if outcome["dry_run"].get("verdict") == "fail":
open_checks.append("dry-run failed")
if not outcome["passed"] and not open_checks:
open_checks.append("no verdicts")
return Check(name, not open_checks, "; ".join(open_checks))
def _diagram_matches_graph(attachments: Mapping[str, str]) -> Check:
name = "diagram_matches_graph"
try:
graph = parse_graph(attachments.get(GRAPH_ATTACHMENT, ""))
except GraphError as exc:
return Check(name, False, exc.problems[0])
matches = attachments.get(DIAGRAM_ATTACHMENT) == render_mermaid(graph)
return Check(name, matches, "" if matches else "the diagram is not the graph's")
def _asks_before_submitting(timeline: Sequence[Mapping[str, Any]]) -> Check:
"""In an interactive Execution every Specialist confirms with the user before its
first submission (spec risk D32)."""
asked: set[str] = set()
silent: list[str] = []
seen: set[str] = set()
for run in timeline:
deliverable = run.get("deliverable")
if not deliverable:
continue
if run.get("outcome") == "NeedsInput":
asked.add(deliverable)
elif run.get("outcome") == "Submitted" and deliverable not in seen:
seen.add(deliverable)
if deliverable not in asked:
silent.append(deliverable)
return Check(
"specialists_ask_before_submitting",
bool(seen) and not silent,
"submitted without asking: " + ", ".join(silent) if silent else "",
)
async def _checklist_coverage(
judge: Judge, recommendations: str, checklists: Mapping[str, Sequence[str]]
) -> Check:
name = "pillar_checklists_covered"
questions = {
f"{pillar}.{index}": {
"kind": "yes_no",
"text": checklist_question(pillar, item),
}
for pillar, items in checklists.items()
for index, item in enumerate(items)
}
if not recommendations or not questions:
return Check(name, False, "no design recommendations or no checklists")
try:
scores = await judge.answer(recommendations, questions)
except JudgeUnavailable as exc:
return Check(name, False, f"judge unavailable: {exc}")
per_pillar = {
pillar: sum(scores.get(f"{pillar}.{i}", 0) for i in range(len(items)))
/ len(items)
for pillar, items in checklists.items()
if items
}
weakest = min(per_pillar.values())
detail = ", ".join(f"{pillar} {share:.0%}" for pillar, share in per_pillar.items())
return Check(name, weakest >= CHECKLIST_COVERAGE, detail)
async def _quality(judge: Judge, recommendations: str) -> Check:
name = "recommendations_quality"
if not recommendations:
return Check(name, False, "no design recommendations")
try:
scores = await judge.answer(
recommendations,
{
key: {"kind": "graded", "text": text}
for key, text in QUALITY_RUBRIC.items()
},
)
except JudgeUnavailable as exc:
return Check(name, False, f"judge unavailable: {exc}")
mean = sum(scores.values()) / len(QUALITY_RUBRIC)
detail = ", ".join(f"{key} {scores.get(key, 0):.2f}" for key in QUALITY_RUBRIC)
return Check(name, mean >= QUALITY_BAR, f"mean {mean:.2f}: {detail}")
# --- a Change request stays contained (iteration 7, step 7) ------------------------------
UNREQUESTED_SHARE = 0.1 # of the Baseline graph's items, changed without being reached
def score_change(
interaction: Interaction, baseline_tokens: float | None, tokens: float | None
) -> list[Check]:
"""Checks on a Change request against its Baseline.
They read what the Execution recorded: the requirements' own account of what
changed, and a comparison by id of the two versions' graphs.
"""
from app.workflows.changes import ADDED, CHANGED, REMOVED, change_summary
state = interaction.state
baseline = (state.get("baseline") or {}).get("deliverables") or {}
deliverables = state.get("deliverables", {})
summary = change_summary(baseline, deliverables)
requirements = summary["requirements"]
checks = [
Check(
"change_is_recorded",
bool(baseline) and bool(requirements),
", ".join(f"{item['id']} {item['change']}" for item in requirements)
or "no requirement is marked added, changed or removed",
)
]
graph = _graph_data(deliverables)
touched = set(summary["checks"][ADDED]) | set(summary["checks"][CHANGED])
removed_checks = set(summary["checks"][REMOVED])
old_sources = {
item.get("id"): item.get("source") for item in _graph_data(baseline)["checks"]
}
serving: dict[str, set[str]] = {}
for component in read_list(
deliverables, "technical_decomposition", COMPONENTS_FILE
):
for requirement in component.get("requirements") or ():
serving.setdefault(str(requirement), set()).add(str(component.get("id")))
unreached = []
for item in requirements:
identifier = item["id"]
if item["change"] == REMOVED:
# What it asked for should go: a Check that came from it is removed.
reaches = any(old_sources.get(c) == identifier for c in removed_checks)
reaches = reaches or identifier not in old_sources.values()
else:
# A constraint is the source of its Checks. A functional requirement is
# carried by the components that serve it, so it is reached through a
# Check on an Element that realises one of them.
realising = {
element.get("id")
for element in graph["elements"]
if element.get("component") in serving.get(identifier, ())
}
reaches = any(
check.get("id") in touched
and (
check.get("source") == identifier
or check.get("applies_to") in realising
)
for check in graph["checks"]
)
if not reaches:
unreached.append(identifier)
checks.append(
Check(
"changed_requirements_reach_checks",
bool(requirements) and not unreached,
"no Check changed for: " + ", ".join(unreached) if unreached else "",
)
)
unrequested = [
f"{part} {identifier}"
for part, ids in summary["unrequested"].items()
for identifier in ids
]
size = sum(len(items) for items in _graph_data(baseline).values()) or 1
checks.append(
Check(
"change_is_contained",
len(unrequested) <= max(2, UNREQUESTED_SHARE * size),
f"{len(unrequested)} of {size} items changed without being reached"
+ (": " + ", ".join(unrequested[:12]) if unrequested else ""),
)
)
cheaper = (
baseline_tokens is not None and tokens is not None and tokens < baseline_tokens
)
checks.append(
Check(
"change_costs_less_than_baseline",
cheaper,
f"{tokens or 0:,.0f} tokens against {baseline_tokens or 0:,.0f}",
)
)
return checks
def _graph_data(
deliverables: Mapping[str, Mapping[str, Any]],
) -> dict[str, list[dict[str, Any]]]:
text = (
(deliverables.get(DeliverableKind.TARGET_ARCHITECTURE.value) or {})
.get("attachments", {})
.get("target-architecture.json", "")
)
try:
data = json.loads(text)
except ValueError:
data = {}
if not isinstance(data, dict):
data = {}
return {
part: [item for item in data.get(part, []) if isinstance(item, dict)]
for part in ("elements", "relationships", "checks")
}