Files
gcp_solution_architecture_a…/eval/scoring.py
Jonathan Boniface 7ee49ab352
Some checks failed
validation / verify (push) Failing after 10s
latest refactoring
2026-10-06 20:22:37 +01:00

208 lines
6.0 KiB
Python

"""Score a finished Execution against a benchmark case.
Each check is a small, explainable rule tied to the upstream Skill's requirements.
A case passes when its score reaches PASS_THRESHOLD.
"""
from __future__ import annotations
import re
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from app.adk.runtime import ToolResult
from app.agent import Interaction
from app.tools.local import mermaid_check
from app.workflows.deliverables import DeliverableKind
PASS_THRESHOLD = 0.8
TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]]
_HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE)
_NUMBERING = re.compile(r"^\d+(?:\.\d+)*\.?\s+")
_DOC_LINK = re.compile(r"https://(?:docs\.)?cloud\.google\.com/\S+")
@dataclass(frozen=True)
class CaseSpec:
id: str
name: str
workflow_request: str
required_products: tuple[str, ...]
scripted_answers: tuple[tuple[str, str], ...]
@dataclass(frozen=True)
class Check:
name: str
passed: bool
detail: str = ""
@dataclass(frozen=True)
class CaseResult:
case_id: str
status: str
checks: tuple[Check, ...]
details: Mapping[str, Any] = field(default_factory=dict)
@property
def score(self) -> float:
return (
sum(c.passed for c in self.checks) / len(self.checks)
if self.checks
else 0.0
)
@property
def passed(self) -> bool:
return self.score >= PASS_THRESHOLD
def load_cases(path: Path) -> list[CaseSpec]:
import json
return [
CaseSpec(
id=item["id"],
name=item["name"],
workflow_request=item["workflow_request"],
required_products=tuple(item["required_products"]),
scripted_answers=tuple(
tuple(pair) for pair in item.get("scripted_answers", [])
),
)
for item in json.loads(path.read_text(encoding="utf-8"))
]
def template_headings(template: Path) -> list[str]:
"""Section headings (level 2+) of a solution-guide template, placeholders removed."""
return [
heading
for _, heading in _HEADING.findall(template.read_text(encoding="utf-8"))
if "[" not in heading
]
def _normalise(text: str) -> str:
return " ".join(_NUMBERING.sub("", text.strip().lower()).split())
def _mentions(text: str, product: str) -> bool:
return (
re.search(rf"(?<![\w/]){re.escape(product.lower())}(?![\w/])", text.lower())
is not None
)
async def score_execution(
case: CaseSpec,
interaction: Interaction,
headings: Sequence[str],
terraform_validate: TerraformValidate,
) -> CaseResult:
deliverables: Mapping[str, Mapping[str, Any]] = interaction.state.get(
"deliverables", {}
)
def content(kind: DeliverableKind) -> str:
item = deliverables.get(kind.value)
return item["content"] if item else ""
def attachments(kind: DeliverableKind) -> Mapping[str, str]:
item = deliverables.get(kind.value)
return item.get("attachments", {}) if item else {}
checks: list[Check] = []
checks.append(
Check(
"completed",
interaction.status == "completed",
"; ".join(interaction.state.get("warnings", [])) or interaction.status,
)
)
guide = content(DeliverableKind.SOLUTION_GUIDE)
guide_headings = {_normalise(h) for _, h in _HEADING.findall(guide)}
missing_sections = [h for h in headings if _normalise(h) not in guide_headings]
checks.append(
Check(
"guide_follows_template",
bool(guide) and not missing_sections,
"missing: " + "; ".join(missing_sections) if missing_sections else "",
)
)
selection = content(DeliverableKind.PRODUCT_SELECTION)
selected_text = selection + "\n" + guide
missing_products = [
p for p in case.required_products if not _mentions(selected_text, p)
]
checks.append(
Check(
"required_products_selected",
bool(selection) and not missing_products,
"missing: " + ", ".join(missing_products) if missing_products else "",
)
)
lowered = selection.lower()
checks.append(
Check(
"selection_has_alternatives",
"alternative" in lowered and "pros" in lowered and "cons" in lowered,
)
)
checks.append(
Check(
"selection_cites_documentation",
bool(_DOC_LINK.search(selection)),
)
)
requirements = content(DeliverableKind.REQUIREMENTS)
leaked = [
product
for product in case.required_products
if _mentions(requirements, product)
and not _mentions(case.workflow_request, product)
]
checks.append(
Check(
"requirements_name_no_products",
bool(requirements) and not leaked,
"names: " + ", ".join(leaked) if leaked else "",
)
)
diagram = attachments(DeliverableKind.ARCHITECTURE_DIAGRAM).get(
"architecture.mmd", ""
)
mermaid = await mermaid_check(diagram)
checks.append(
Check(
"diagram_is_valid_mermaid",
bool(diagram) and mermaid.message.startswith("Mermaid check passed"),
mermaid.message,
)
)
terraform_files = {
name.removeprefix("terraform/"): text
for name, text in attachments(DeliverableKind.DEPLOYMENT_GUIDANCE).items()
if name.endswith((".tf", ".tfvars", ".tf.json"))
}
if terraform_files:
validation = await terraform_validate(terraform_files)
terraform_ok = validation.message.startswith("terraform validate passed")
terraform_detail = validation.message[:1500]
else:
terraform_ok, terraform_detail = False, "no Terraform files attached"
checks.append(Check("terraform_validates", terraform_ok, terraform_detail))
return CaseResult(case.id, interaction.status, tuple(checks))