This commit is contained in:
207
eval/scoring.py
Normal file
207
eval/scoring.py
Normal file
@@ -0,0 +1,207 @@
|
||||
"""Score a finished Execution against a benchmark case.
|
||||
|
||||
Each check is a small, explainable rule tied to the upstream Skill's requirements.
|
||||
A case passes when its score reaches PASS_THRESHOLD.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.adk.runtime import ToolResult
|
||||
from app.agent import Interaction
|
||||
from app.tools.local import mermaid_check
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
PASS_THRESHOLD = 0.8
|
||||
TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]]
|
||||
|
||||
_HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE)
|
||||
_NUMBERING = re.compile(r"^\d+(?:\.\d+)*\.?\s+")
|
||||
_DOC_LINK = re.compile(r"https://(?:docs\.)?cloud\.google\.com/\S+")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CaseSpec:
|
||||
id: str
|
||||
name: str
|
||||
workflow_request: str
|
||||
required_products: tuple[str, ...]
|
||||
scripted_answers: tuple[tuple[str, str], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Check:
|
||||
name: str
|
||||
passed: bool
|
||||
detail: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CaseResult:
|
||||
case_id: str
|
||||
status: str
|
||||
checks: tuple[Check, ...]
|
||||
details: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def score(self) -> float:
|
||||
return (
|
||||
sum(c.passed for c in self.checks) / len(self.checks)
|
||||
if self.checks
|
||||
else 0.0
|
||||
)
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
return self.score >= PASS_THRESHOLD
|
||||
|
||||
|
||||
def load_cases(path: Path) -> list[CaseSpec]:
|
||||
import json
|
||||
|
||||
return [
|
||||
CaseSpec(
|
||||
id=item["id"],
|
||||
name=item["name"],
|
||||
workflow_request=item["workflow_request"],
|
||||
required_products=tuple(item["required_products"]),
|
||||
scripted_answers=tuple(
|
||||
tuple(pair) for pair in item.get("scripted_answers", [])
|
||||
),
|
||||
)
|
||||
for item in json.loads(path.read_text(encoding="utf-8"))
|
||||
]
|
||||
|
||||
|
||||
def template_headings(template: Path) -> list[str]:
|
||||
"""Section headings (level 2+) of a solution-guide template, placeholders removed."""
|
||||
return [
|
||||
heading
|
||||
for _, heading in _HEADING.findall(template.read_text(encoding="utf-8"))
|
||||
if "[" not in heading
|
||||
]
|
||||
|
||||
|
||||
def _normalise(text: str) -> str:
|
||||
return " ".join(_NUMBERING.sub("", text.strip().lower()).split())
|
||||
|
||||
|
||||
def _mentions(text: str, product: str) -> bool:
|
||||
return (
|
||||
re.search(rf"(?<![\w/]){re.escape(product.lower())}(?![\w/])", text.lower())
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
async def score_execution(
|
||||
case: CaseSpec,
|
||||
interaction: Interaction,
|
||||
headings: Sequence[str],
|
||||
terraform_validate: TerraformValidate,
|
||||
) -> CaseResult:
|
||||
deliverables: Mapping[str, Mapping[str, Any]] = interaction.state.get(
|
||||
"deliverables", {}
|
||||
)
|
||||
|
||||
def content(kind: DeliverableKind) -> str:
|
||||
item = deliverables.get(kind.value)
|
||||
return item["content"] if item else ""
|
||||
|
||||
def attachments(kind: DeliverableKind) -> Mapping[str, str]:
|
||||
item = deliverables.get(kind.value)
|
||||
return item.get("attachments", {}) if item else {}
|
||||
|
||||
checks: list[Check] = []
|
||||
|
||||
checks.append(
|
||||
Check(
|
||||
"completed",
|
||||
interaction.status == "completed",
|
||||
"; ".join(interaction.state.get("warnings", [])) or interaction.status,
|
||||
)
|
||||
)
|
||||
|
||||
guide = content(DeliverableKind.SOLUTION_GUIDE)
|
||||
guide_headings = {_normalise(h) for _, h in _HEADING.findall(guide)}
|
||||
missing_sections = [h for h in headings if _normalise(h) not in guide_headings]
|
||||
checks.append(
|
||||
Check(
|
||||
"guide_follows_template",
|
||||
bool(guide) and not missing_sections,
|
||||
"missing: " + "; ".join(missing_sections) if missing_sections else "",
|
||||
)
|
||||
)
|
||||
|
||||
selection = content(DeliverableKind.PRODUCT_SELECTION)
|
||||
selected_text = selection + "\n" + guide
|
||||
missing_products = [
|
||||
p for p in case.required_products if not _mentions(selected_text, p)
|
||||
]
|
||||
checks.append(
|
||||
Check(
|
||||
"required_products_selected",
|
||||
bool(selection) and not missing_products,
|
||||
"missing: " + ", ".join(missing_products) if missing_products else "",
|
||||
)
|
||||
)
|
||||
|
||||
lowered = selection.lower()
|
||||
checks.append(
|
||||
Check(
|
||||
"selection_has_alternatives",
|
||||
"alternative" in lowered and "pros" in lowered and "cons" in lowered,
|
||||
)
|
||||
)
|
||||
checks.append(
|
||||
Check(
|
||||
"selection_cites_documentation",
|
||||
bool(_DOC_LINK.search(selection)),
|
||||
)
|
||||
)
|
||||
|
||||
requirements = content(DeliverableKind.REQUIREMENTS)
|
||||
leaked = [
|
||||
product
|
||||
for product in case.required_products
|
||||
if _mentions(requirements, product)
|
||||
and not _mentions(case.workflow_request, product)
|
||||
]
|
||||
checks.append(
|
||||
Check(
|
||||
"requirements_name_no_products",
|
||||
bool(requirements) and not leaked,
|
||||
"names: " + ", ".join(leaked) if leaked else "",
|
||||
)
|
||||
)
|
||||
|
||||
diagram = attachments(DeliverableKind.ARCHITECTURE_DIAGRAM).get(
|
||||
"architecture.mmd", ""
|
||||
)
|
||||
mermaid = await mermaid_check(diagram)
|
||||
checks.append(
|
||||
Check(
|
||||
"diagram_is_valid_mermaid",
|
||||
bool(diagram) and mermaid.message.startswith("Mermaid check passed"),
|
||||
mermaid.message,
|
||||
)
|
||||
)
|
||||
|
||||
terraform_files = {
|
||||
name.removeprefix("terraform/"): text
|
||||
for name, text in attachments(DeliverableKind.DEPLOYMENT_GUIDANCE).items()
|
||||
if name.endswith((".tf", ".tfvars", ".tf.json"))
|
||||
}
|
||||
if terraform_files:
|
||||
validation = await terraform_validate(terraform_files)
|
||||
terraform_ok = validation.message.startswith("terraform validate passed")
|
||||
terraform_detail = validation.message[:1500]
|
||||
else:
|
||||
terraform_ok, terraform_detail = False, "no Terraform files attached"
|
||||
checks.append(Check("terraform_validates", terraform_ok, terraform_detail))
|
||||
|
||||
return CaseResult(case.id, interaction.status, tuple(checks))
|
||||
Reference in New Issue
Block a user