Files
gcp_solution_architecture_a…/eval/scoring.py
Jonathan Boniface 5bb22d984e
Some checks failed
validation / verify (push) Failing after 8s
fixed multile issues
2026-10-07 17:33:36 +01:00

438 lines
15 KiB
Python

"""Score a finished Execution against a benchmark case.
Each check is a small, explainable rule tied to the upstream Skill's requirements.
A case passes when its score reaches PASS_THRESHOLD.
Structural checks always run. Judged checks (pillar checklist coverage, recommendation
quality) run when a judge is given, and are what lets a better or worse Skill move the
score. Thresholds on revisions, seconds and tokens turn a slower or costlier run into a
reported regression without changing the score.
"""
from __future__ import annotations
import json
import re
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from app.adk.runtime import ToolResult
from app.agent import Interaction
from app.tools.local import mermaid_check
from app.workflows.chain import (
COMPONENTS_FILE,
PRODUCTS_FILE,
RECOMMENDATIONS_FILE,
REQUIREMENTS_FILE,
VERDICTS_FILE,
read_graph,
read_list,
validation_outcome,
)
from app.workflows.deliverables import DeliverableKind
from app.workflows.target_architecture import (
DIAGRAM_ATTACHMENT,
GRAPH_ATTACHMENT,
GraphError,
chain_gaps,
parse_graph,
render_mermaid,
)
from eval.judge import Judge, JudgeUnavailable
PASS_THRESHOLD = 0.8
CHECKLIST_COVERAGE = 0.7 # share of a pillar Skill's checklist the design must address
QUALITY_BAR = 0.67 # mean of the graded rubric: "mostly" or better
THRESHOLD_METRICS = ("revisions", "seconds", "tokens")
# What a reviewer would ask of design recommendations, graded 0 to 1 each.
QUALITY_RUBRIC: Mapping[str, str] = {
"specific": "Are the recommendations specific to this workload and its stated "
"requirements, as opposed to generic best practice that would fit any workload?",
"actionable": "Does each recommendation name the concrete setting, feature or "
"configuration to apply, so an engineer could act on it without more research?",
"trade_offs": "Where pillars pull in different directions (for example cost "
"against reliability), is the trade-off named and a resolution recommended?",
"traceable": "Can each recommendation be traced to a requirement or constraint "
"of the workload?",
}
TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]]
_HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE)
_NUMBERING = re.compile(r"^\d+(?:\.\d+)*\.?\s+")
_DOC_LINK = re.compile(r"https://(?:docs\.)?cloud\.google\.com/\S+")
@dataclass(frozen=True)
class CaseSpec:
id: str
name: str
workflow_request: str
required_products: tuple[str, ...]
scripted_answers: tuple[tuple[str, str], ...]
# Run with a user in the loop (scripted): Approvals and questions are answered
# turn by turn, and Specialists are checked for asking before they submit.
interactive: bool = False
# Upper bounds for "revisions", "seconds" and "tokens"; exceeding one is a regression.
thresholds: Mapping[str, float] = field(default_factory=dict)
@dataclass(frozen=True)
class Check:
name: str
passed: bool
detail: str = ""
@dataclass(frozen=True)
class CaseResult:
case_id: str
status: str
checks: tuple[Check, ...]
details: Mapping[str, Any] = field(default_factory=dict)
@property
def score(self) -> float:
return (
sum(c.passed for c in self.checks) / len(self.checks)
if self.checks
else 0.0
)
@property
def passed(self) -> bool:
return self.score >= PASS_THRESHOLD
def load_cases(path: Path) -> list[CaseSpec]:
return [
CaseSpec(
id=item["id"],
name=item["name"],
workflow_request=item["workflow_request"],
required_products=tuple(item["required_products"]),
scripted_answers=tuple(
tuple(pair) for pair in item.get("scripted_answers", [])
),
interactive=bool(item.get("interactive", False)),
thresholds=dict(item.get("thresholds", {})),
)
for item in json.loads(path.read_text(encoding="utf-8"))
]
def template_headings(template: Path) -> list[str]:
"""Section headings (level 2+) of a solution-guide template, placeholders removed."""
return [
heading
for _, heading in _HEADING.findall(template.read_text(encoding="utf-8"))
if "[" not in heading
]
def checklist_items(skill_file: Path) -> list[str]:
"""The bullet items under a pillar Skill's "Validation checklist" heading."""
text = skill_file.read_text(encoding="utf-8")
match = re.search(
r"^##\s+Validation checklist\s*$(.*?)(?=^##\s|\Z)",
text,
re.MULTILINE | re.DOTALL,
)
if not match:
return []
# Bullets may nest: a bullet with deeper bullets under it is a category, and the
# items to check are the leaves, each prefixed with its category.
bullets: list[tuple[int, str]] = []
for line in match.group(1).splitlines():
bullet = re.match(r"^(\s*)[-*]\s+(.*)$", line)
if bullet:
text = re.sub(r"^\[[ xX]?\]\s*", "", bullet.group(2).strip())
bullets.append((len(bullet.group(1)), text.replace("**", "")))
elif bullets and line.startswith(" ") and line.strip():
indent, text = bullets[-1]
bullets[-1] = (indent, f"{text} {line.strip()}")
items: list[str] = []
category = ""
for index, (indent, text) in enumerate(bullets):
has_children = index + 1 < len(bullets) and bullets[index + 1][0] > indent
if has_children:
category = text.rstrip(":").strip()
elif indent == 0:
category = ""
items.append(text)
else:
items.append(f"{category}: {text}" if category else text)
return items
def regressions(case: CaseSpec, measured: Mapping[str, float | None]) -> list[str]:
"""Each threshold the run exceeded, as "metric: measured > limit"."""
over = []
for metric in THRESHOLD_METRICS:
limit = case.thresholds.get(metric)
value = measured.get(metric)
if limit is not None and value is not None and value > limit:
over.append(f"{metric}: {value:g} > {limit:g}")
return over
def _normalise(text: str) -> str:
return " ".join(_NUMBERING.sub("", text.strip().lower()).split())
def _mentions(text: str, product: str) -> bool:
return (
re.search(rf"(?<![\w/]){re.escape(product.lower())}(?![\w/])", text.lower())
is not None
)
async def score_execution(
case: CaseSpec,
interaction: Interaction,
headings: Sequence[str],
terraform_validate: TerraformValidate,
*,
judge: Judge | None = None,
pillar_checklists: Mapping[str, Sequence[str]] | None = None,
timeline: Sequence[Mapping[str, Any]] = (),
) -> CaseResult:
deliverables: Mapping[str, Mapping[str, Any]] = interaction.state.get(
"deliverables", {}
)
def content(kind: DeliverableKind) -> str:
item = deliverables.get(kind.value)
return item["content"] if item else ""
def attachments(kind: DeliverableKind) -> Mapping[str, str]:
item = deliverables.get(kind.value)
return item.get("attachments", {}) if item else {}
checks: list[Check] = []
checks.append(
Check(
"completed",
interaction.status == "completed",
"; ".join(interaction.state.get("warnings", [])) or interaction.status,
)
)
guide = content(DeliverableKind.SOLUTION_GUIDE)
guide_headings = {_normalise(h) for _, h in _HEADING.findall(guide)}
missing_sections = [h for h in headings if _normalise(h) not in guide_headings]
checks.append(
Check(
"guide_follows_template",
bool(guide) and not missing_sections,
"missing: " + "; ".join(missing_sections) if missing_sections else "",
)
)
selection = content(DeliverableKind.PRODUCT_SELECTION)
selected_text = selection + "\n" + guide
missing_products = [
p for p in case.required_products if not _mentions(selected_text, p)
]
checks.append(
Check(
"required_products_selected",
bool(selection) and not missing_products,
"missing: " + ", ".join(missing_products) if missing_products else "",
)
)
lowered = selection.lower()
checks.append(
Check(
"selection_has_alternatives",
"alternative" in lowered and "pros" in lowered and "cons" in lowered,
)
)
checks.append(
Check(
"selection_cites_documentation",
bool(_DOC_LINK.search(selection)),
)
)
requirements = content(DeliverableKind.REQUIREMENTS)
leaked = [
product
for product in case.required_products
if _mentions(requirements, product)
and not _mentions(case.workflow_request, product)
]
checks.append(
Check(
"requirements_name_no_products",
bool(requirements) and not leaked,
"names: " + ", ".join(leaked) if leaked else "",
)
)
diagram = attachments(DeliverableKind.TARGET_ARCHITECTURE).get(
"architecture.mmd", ""
)
mermaid = await mermaid_check(diagram)
checks.append(
Check(
"diagram_is_valid_mermaid",
bool(diagram) and mermaid.message.startswith("Mermaid check passed"),
mermaid.message,
)
)
terraform_files = {
name.removeprefix("terraform/"): text
for name, text in attachments(DeliverableKind.DEPLOYMENT_GUIDANCE).items()
if name.endswith((".tf", ".tfvars", ".tf.json"))
}
if terraform_files:
validation = await terraform_validate(terraform_files)
terraform_ok = validation.message.startswith("terraform validate passed")
terraform_detail = validation.message[:1500]
else:
terraform_ok, terraform_detail = False, "no Terraform files attached"
checks.append(Check("terraform_validates", terraform_ok, terraform_detail))
checks.append(_chain_check(deliverables))
checks.append(_verdicts_check(attachments(DeliverableKind.VALIDATION_RESULTS)))
checks.append(
_diagram_matches_graph(attachments(DeliverableKind.TARGET_ARCHITECTURE))
)
if case.interactive:
checks.append(_asks_before_submitting(timeline))
if judge is not None:
recommendations = content(DeliverableKind.DESIGN_RECOMMENDATIONS)
checks.append(
await _checklist_coverage(judge, recommendations, pillar_checklists or {})
)
checks.append(await _quality(judge, recommendations))
return CaseResult(case.id, interaction.status, tuple(checks))
def _chain_check(deliverables: Mapping[str, Mapping[str, Any]]) -> Check:
"""The Requirements chain holds end to end: every requirement reaches a Check,
directly or through a component's Element, and nothing along it is missing."""
name = "requirements_chain_complete"
graph = read_graph(deliverables)
if graph is None:
return Check(name, False, "no well-formed Target architecture")
gaps = chain_gaps(
graph,
requirements=read_list(deliverables, "requirements", REQUIREMENTS_FILE),
components=read_list(deliverables, "technical_decomposition", COMPONENTS_FILE),
products=read_list(deliverables, "product_selection", PRODUCTS_FILE),
recommendations=read_list(
deliverables, "design_recommendations", RECOMMENDATIONS_FILE
),
)
return Check(name, not gaps, "; ".join(gaps))
def _verdicts_check(attachments: Mapping[str, str]) -> Check:
"""Validation ended with every Check passed and the dry-run not failed."""
name = "every_check_passes"
text = attachments.get(VERDICTS_FILE)
if text is None:
return Check(name, False, f"no {VERDICTS_FILE}")
outcome = validation_outcome(text)
open_checks = [f"{item['check']} failed" for item in outcome["failed"]] + [
f"{item['check']} not verifiable" for item in outcome["not_verifiable"]
]
if outcome["dry_run"].get("verdict") == "fail":
open_checks.append("dry-run failed")
if not outcome["passed"] and not open_checks:
open_checks.append("no verdicts")
return Check(name, not open_checks, "; ".join(open_checks))
def _diagram_matches_graph(attachments: Mapping[str, str]) -> Check:
name = "diagram_matches_graph"
try:
graph = parse_graph(attachments.get(GRAPH_ATTACHMENT, ""))
except GraphError as exc:
return Check(name, False, exc.problems[0])
matches = attachments.get(DIAGRAM_ATTACHMENT) == render_mermaid(graph)
return Check(name, matches, "" if matches else "the diagram is not the graph's")
def _asks_before_submitting(timeline: Sequence[Mapping[str, Any]]) -> Check:
"""In an interactive Execution every Specialist confirms with the user before its
first submission (spec risk D32)."""
asked: set[str] = set()
silent: list[str] = []
seen: set[str] = set()
for run in timeline:
deliverable = run.get("deliverable")
if not deliverable:
continue
if run.get("outcome") == "NeedsInput":
asked.add(deliverable)
elif run.get("outcome") == "Submitted" and deliverable not in seen:
seen.add(deliverable)
if deliverable not in asked:
silent.append(deliverable)
return Check(
"specialists_ask_before_submitting",
bool(seen) and not silent,
"submitted without asking: " + ", ".join(silent) if silent else "",
)
async def _checklist_coverage(
judge: Judge, recommendations: str, checklists: Mapping[str, Sequence[str]]
) -> Check:
name = "pillar_checklists_covered"
questions = {
f"{pillar}.{index}": {
"kind": "yes_no",
"text": f"Do the recommendations address this {pillar} checklist item, "
f"either by recommending it or by explaining why it does not apply? {item}",
}
for pillar, items in checklists.items()
for index, item in enumerate(items)
}
if not recommendations or not questions:
return Check(name, False, "no design recommendations or no checklists")
try:
scores = await judge.answer(recommendations, questions)
except JudgeUnavailable as exc:
return Check(name, False, f"judge unavailable: {exc}")
per_pillar = {
pillar: sum(scores.get(f"{pillar}.{i}", 0) for i in range(len(items)))
/ len(items)
for pillar, items in checklists.items()
if items
}
weakest = min(per_pillar.values())
detail = ", ".join(f"{pillar} {share:.0%}" for pillar, share in per_pillar.items())
return Check(name, weakest >= CHECKLIST_COVERAGE, detail)
async def _quality(judge: Judge, recommendations: str) -> Check:
name = "recommendations_quality"
if not recommendations:
return Check(name, False, "no design recommendations")
try:
scores = await judge.answer(
recommendations,
{
key: {"kind": "graded", "text": text}
for key, text in QUALITY_RUBRIC.items()
},
)
except JudgeUnavailable as exc:
return Check(name, False, f"judge unavailable: {exc}")
mean = sum(scores.values()) / len(QUALITY_RUBRIC)
detail = ", ".join(f"{key} {scores.get(key, 0):.2f}" for key in QUALITY_RUBRIC)
return Check(name, mean >= QUALITY_BAR, f"mean {mean:.2f}: {detail}")