"""Score a finished Execution against a benchmark case. Each check is a small, explainable rule tied to the upstream Skill's requirements. A case passes when its score reaches PASS_THRESHOLD. """ from __future__ import annotations import re from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path from typing import Any from app.adk.runtime import ToolResult from app.agent import Interaction from app.tools.local import mermaid_check from app.workflows.deliverables import DeliverableKind PASS_THRESHOLD = 0.8 TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]] _HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE) _NUMBERING = re.compile(r"^\d+(?:\.\d+)*\.?\s+") _DOC_LINK = re.compile(r"https://(?:docs\.)?cloud\.google\.com/\S+") @dataclass(frozen=True) class CaseSpec: id: str name: str workflow_request: str required_products: tuple[str, ...] scripted_answers: tuple[tuple[str, str], ...] @dataclass(frozen=True) class Check: name: str passed: bool detail: str = "" @dataclass(frozen=True) class CaseResult: case_id: str status: str checks: tuple[Check, ...] details: Mapping[str, Any] = field(default_factory=dict) @property def score(self) -> float: return ( sum(c.passed for c in self.checks) / len(self.checks) if self.checks else 0.0 ) @property def passed(self) -> bool: return self.score >= PASS_THRESHOLD def load_cases(path: Path) -> list[CaseSpec]: import json return [ CaseSpec( id=item["id"], name=item["name"], workflow_request=item["workflow_request"], required_products=tuple(item["required_products"]), scripted_answers=tuple( tuple(pair) for pair in item.get("scripted_answers", []) ), ) for item in json.loads(path.read_text(encoding="utf-8")) ] def template_headings(template: Path) -> list[str]: """Section headings (level 2+) of a solution-guide template, placeholders removed.""" return [ heading for _, heading in _HEADING.findall(template.read_text(encoding="utf-8")) if "[" not in heading ] def _normalise(text: str) -> str: return " ".join(_NUMBERING.sub("", text.strip().lower()).split()) def _mentions(text: str, product: str) -> bool: return ( re.search(rf"(? CaseResult: deliverables: Mapping[str, Mapping[str, Any]] = interaction.state.get( "deliverables", {} ) def content(kind: DeliverableKind) -> str: item = deliverables.get(kind.value) return item["content"] if item else "" def attachments(kind: DeliverableKind) -> Mapping[str, str]: item = deliverables.get(kind.value) return item.get("attachments", {}) if item else {} checks: list[Check] = [] checks.append( Check( "completed", interaction.status == "completed", "; ".join(interaction.state.get("warnings", [])) or interaction.status, ) ) guide = content(DeliverableKind.SOLUTION_GUIDE) guide_headings = {_normalise(h) for _, h in _HEADING.findall(guide)} missing_sections = [h for h in headings if _normalise(h) not in guide_headings] checks.append( Check( "guide_follows_template", bool(guide) and not missing_sections, "missing: " + "; ".join(missing_sections) if missing_sections else "", ) ) selection = content(DeliverableKind.PRODUCT_SELECTION) selected_text = selection + "\n" + guide missing_products = [ p for p in case.required_products if not _mentions(selected_text, p) ] checks.append( Check( "required_products_selected", bool(selection) and not missing_products, "missing: " + ", ".join(missing_products) if missing_products else "", ) ) lowered = selection.lower() checks.append( Check( "selection_has_alternatives", "alternative" in lowered and "pros" in lowered and "cons" in lowered, ) ) checks.append( Check( "selection_cites_documentation", bool(_DOC_LINK.search(selection)), ) ) requirements = content(DeliverableKind.REQUIREMENTS) leaked = [ product for product in case.required_products if _mentions(requirements, product) and not _mentions(case.workflow_request, product) ] checks.append( Check( "requirements_name_no_products", bool(requirements) and not leaked, "names: " + ", ".join(leaked) if leaked else "", ) ) diagram = attachments(DeliverableKind.ARCHITECTURE_DIAGRAM).get( "architecture.mmd", "" ) mermaid = await mermaid_check(diagram) checks.append( Check( "diagram_is_valid_mermaid", bool(diagram) and mermaid.message.startswith("Mermaid check passed"), mermaid.message, ) ) terraform_files = { name.removeprefix("terraform/"): text for name, text in attachments(DeliverableKind.DEPLOYMENT_GUIDANCE).items() if name.endswith((".tf", ".tfvars", ".tf.json")) } if terraform_files: validation = await terraform_validate(terraform_files) terraform_ok = validation.message.startswith("terraform validate passed") terraform_detail = validation.message[:1500] else: terraform_ok, terraform_detail = False, "no Terraform files attached" checks.append(Check("terraform_validates", terraform_ok, terraform_detail)) return CaseResult(case.id, interaction.status, tuple(checks))