208 lines
6.0 KiB
Python
208 lines
6.0 KiB
Python
"""Score a finished Execution against a benchmark case.
|
|
|
|
Each check is a small, explainable rule tied to the upstream Skill's requirements.
|
|
A case passes when its score reaches PASS_THRESHOLD.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from app.adk.runtime import ToolResult
|
|
from app.agent import Interaction
|
|
from app.tools.local import mermaid_check
|
|
from app.workflows.deliverables import DeliverableKind
|
|
|
|
PASS_THRESHOLD = 0.8
|
|
TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]]
|
|
|
|
_HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE)
|
|
_NUMBERING = re.compile(r"^\d+(?:\.\d+)*\.?\s+")
|
|
_DOC_LINK = re.compile(r"https://(?:docs\.)?cloud\.google\.com/\S+")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CaseSpec:
|
|
id: str
|
|
name: str
|
|
workflow_request: str
|
|
required_products: tuple[str, ...]
|
|
scripted_answers: tuple[tuple[str, str], ...]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Check:
|
|
name: str
|
|
passed: bool
|
|
detail: str = ""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CaseResult:
|
|
case_id: str
|
|
status: str
|
|
checks: tuple[Check, ...]
|
|
details: Mapping[str, Any] = field(default_factory=dict)
|
|
|
|
@property
|
|
def score(self) -> float:
|
|
return (
|
|
sum(c.passed for c in self.checks) / len(self.checks)
|
|
if self.checks
|
|
else 0.0
|
|
)
|
|
|
|
@property
|
|
def passed(self) -> bool:
|
|
return self.score >= PASS_THRESHOLD
|
|
|
|
|
|
def load_cases(path: Path) -> list[CaseSpec]:
|
|
import json
|
|
|
|
return [
|
|
CaseSpec(
|
|
id=item["id"],
|
|
name=item["name"],
|
|
workflow_request=item["workflow_request"],
|
|
required_products=tuple(item["required_products"]),
|
|
scripted_answers=tuple(
|
|
tuple(pair) for pair in item.get("scripted_answers", [])
|
|
),
|
|
)
|
|
for item in json.loads(path.read_text(encoding="utf-8"))
|
|
]
|
|
|
|
|
|
def template_headings(template: Path) -> list[str]:
|
|
"""Section headings (level 2+) of a solution-guide template, placeholders removed."""
|
|
return [
|
|
heading
|
|
for _, heading in _HEADING.findall(template.read_text(encoding="utf-8"))
|
|
if "[" not in heading
|
|
]
|
|
|
|
|
|
def _normalise(text: str) -> str:
|
|
return " ".join(_NUMBERING.sub("", text.strip().lower()).split())
|
|
|
|
|
|
def _mentions(text: str, product: str) -> bool:
|
|
return (
|
|
re.search(rf"(?<![\w/]){re.escape(product.lower())}(?![\w/])", text.lower())
|
|
is not None
|
|
)
|
|
|
|
|
|
async def score_execution(
|
|
case: CaseSpec,
|
|
interaction: Interaction,
|
|
headings: Sequence[str],
|
|
terraform_validate: TerraformValidate,
|
|
) -> CaseResult:
|
|
deliverables: Mapping[str, Mapping[str, Any]] = interaction.state.get(
|
|
"deliverables", {}
|
|
)
|
|
|
|
def content(kind: DeliverableKind) -> str:
|
|
item = deliverables.get(kind.value)
|
|
return item["content"] if item else ""
|
|
|
|
def attachments(kind: DeliverableKind) -> Mapping[str, str]:
|
|
item = deliverables.get(kind.value)
|
|
return item.get("attachments", {}) if item else {}
|
|
|
|
checks: list[Check] = []
|
|
|
|
checks.append(
|
|
Check(
|
|
"completed",
|
|
interaction.status == "completed",
|
|
"; ".join(interaction.state.get("warnings", [])) or interaction.status,
|
|
)
|
|
)
|
|
|
|
guide = content(DeliverableKind.SOLUTION_GUIDE)
|
|
guide_headings = {_normalise(h) for _, h in _HEADING.findall(guide)}
|
|
missing_sections = [h for h in headings if _normalise(h) not in guide_headings]
|
|
checks.append(
|
|
Check(
|
|
"guide_follows_template",
|
|
bool(guide) and not missing_sections,
|
|
"missing: " + "; ".join(missing_sections) if missing_sections else "",
|
|
)
|
|
)
|
|
|
|
selection = content(DeliverableKind.PRODUCT_SELECTION)
|
|
selected_text = selection + "\n" + guide
|
|
missing_products = [
|
|
p for p in case.required_products if not _mentions(selected_text, p)
|
|
]
|
|
checks.append(
|
|
Check(
|
|
"required_products_selected",
|
|
bool(selection) and not missing_products,
|
|
"missing: " + ", ".join(missing_products) if missing_products else "",
|
|
)
|
|
)
|
|
|
|
lowered = selection.lower()
|
|
checks.append(
|
|
Check(
|
|
"selection_has_alternatives",
|
|
"alternative" in lowered and "pros" in lowered and "cons" in lowered,
|
|
)
|
|
)
|
|
checks.append(
|
|
Check(
|
|
"selection_cites_documentation",
|
|
bool(_DOC_LINK.search(selection)),
|
|
)
|
|
)
|
|
|
|
requirements = content(DeliverableKind.REQUIREMENTS)
|
|
leaked = [
|
|
product
|
|
for product in case.required_products
|
|
if _mentions(requirements, product)
|
|
and not _mentions(case.workflow_request, product)
|
|
]
|
|
checks.append(
|
|
Check(
|
|
"requirements_name_no_products",
|
|
bool(requirements) and not leaked,
|
|
"names: " + ", ".join(leaked) if leaked else "",
|
|
)
|
|
)
|
|
|
|
diagram = attachments(DeliverableKind.ARCHITECTURE_DIAGRAM).get(
|
|
"architecture.mmd", ""
|
|
)
|
|
mermaid = await mermaid_check(diagram)
|
|
checks.append(
|
|
Check(
|
|
"diagram_is_valid_mermaid",
|
|
bool(diagram) and mermaid.message.startswith("Mermaid check passed"),
|
|
mermaid.message,
|
|
)
|
|
)
|
|
|
|
terraform_files = {
|
|
name.removeprefix("terraform/"): text
|
|
for name, text in attachments(DeliverableKind.DEPLOYMENT_GUIDANCE).items()
|
|
if name.endswith((".tf", ".tfvars", ".tf.json"))
|
|
}
|
|
if terraform_files:
|
|
validation = await terraform_validate(terraform_files)
|
|
terraform_ok = validation.message.startswith("terraform validate passed")
|
|
terraform_detail = validation.message[:1500]
|
|
else:
|
|
terraform_ok, terraform_detail = False, "no Terraform files attached"
|
|
checks.append(Check("terraform_validates", terraform_ok, terraform_detail))
|
|
|
|
return CaseResult(case.id, interaction.status, tuple(checks))
|