"""Score a finished Execution against a benchmark case. Each check is a small, explainable rule tied to the upstream Skill's requirements. A case passes when its score reaches PASS_THRESHOLD. Structural checks always run. Judged checks (pillar checklist coverage, recommendation quality) run when a judge is given, and are what lets a better or worse Skill move the score. Thresholds on revisions, seconds and tokens turn a slower or costlier run into a reported regression without changing the score. """ from __future__ import annotations import json import re from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path from typing import Any from app.adk.runtime import ToolResult from app.agent import Interaction from app.tools.assessment import QUALITY_BAR, QUALITY_RUBRIC, checklist_question from app.tools.local import mermaid_check from app.workflows.chain import ( COMPONENTS_FILE, PRODUCTS_FILE, RECOMMENDATIONS_FILE, REQUIREMENTS_FILE, VERDICTS_FILE, read_graph, read_list, validation_outcome, ) from app.workflows.deliverables import DeliverableKind from app.workflows.target_architecture import ( DIAGRAM_ATTACHMENT, GRAPH_ATTACHMENT, GraphError, chain_gaps, parse_graph, render_mermaid, ) from eval.judge import Judge, JudgeUnavailable PASS_THRESHOLD = 0.8 CHECKLIST_COVERAGE = 0.7 # share of a pillar Skill's checklist the design must address THRESHOLD_METRICS = ("revisions", "seconds", "tokens") TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]] _HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE) _NUMBERING = re.compile(r"^\d+(?:\.\d+)*\.?\s+") _DOC_LINK = re.compile(r"https://(?:docs\.)?cloud\.google\.com/\S+") @dataclass(frozen=True) class CaseSpec: id: str name: str workflow_request: str required_products: tuple[str, ...] scripted_answers: tuple[tuple[str, ...], ...] # (pattern, answer[, asker]) # Run with a user in the loop (scripted): Approvals and questions are answered # turn by turn, and Specialists are checked for asking before they submit. interactive: bool = False # Upper bounds for "revisions", "seconds" and "tokens"; exceeding one is a regression. thresholds: Mapping[str, float] = field(default_factory=dict) # A change case: the request is run to completion, then this statement of what # has changed is run as a Change request against it (ADR-0006). change: str = "" # Overlay Skills the case's Executions name (ADR-0004). overlay_skills: tuple[str, ...] = () # Whether the case's Executions opt into published Terraform modules. terraform_modules: bool = False # What the infrastructure code must and must not contain, as regular # expressions over all of it: how a case checks that standards from an overlay # Skill reached the code. Each is {"name", "require"} or {"name", "forbid"}. code_standards: tuple[Mapping[str, str], ...] = () @dataclass(frozen=True) class Check: name: str passed: bool detail: str = "" @dataclass(frozen=True) class CaseResult: case_id: str status: str checks: tuple[Check, ...] details: Mapping[str, Any] = field(default_factory=dict) @property def score(self) -> float: return ( sum(c.passed for c in self.checks) / len(self.checks) if self.checks else 0.0 ) @property def passed(self) -> bool: return self.score >= PASS_THRESHOLD def load_cases(path: Path) -> list[CaseSpec]: return [ CaseSpec( id=item["id"], name=item["name"], workflow_request=item["workflow_request"], required_products=tuple(item["required_products"]), scripted_answers=tuple( tuple(pair) for pair in item.get("scripted_answers", []) ), interactive=bool(item.get("interactive", False)), thresholds=dict(item.get("thresholds", {})), change=str(item.get("change", "")), overlay_skills=tuple(item.get("overlay_skills", ())), terraform_modules=bool(item.get("terraform_modules", False)), code_standards=tuple(item.get("code_standards", ())), ) for item in json.loads(path.read_text(encoding="utf-8")) ] def template_headings(template: Path) -> list[str]: """Section headings (level 2+) of a solution-guide template, placeholders removed.""" return [ heading for _, heading in _HEADING.findall(template.read_text(encoding="utf-8")) if "[" not in heading ] def checklist_items(skill_file: Path) -> list[str]: """The bullet items under a pillar Skill's "Validation checklist" heading.""" text = skill_file.read_text(encoding="utf-8") match = re.search( r"^##\s+Validation checklist\s*$(.*?)(?=^##\s|\Z)", text, re.MULTILINE | re.DOTALL, ) if not match: return [] # Bullets may nest: a bullet with deeper bullets under it is a category, and the # items to check are the leaves, each prefixed with its category. bullets: list[tuple[int, str]] = [] for line in match.group(1).splitlines(): bullet = re.match(r"^(\s*)[-*]\s+(.*)$", line) if bullet: text = re.sub(r"^\[[ xX]?\]\s*", "", bullet.group(2).strip()) bullets.append((len(bullet.group(1)), text.replace("**", ""))) elif bullets and line.startswith(" ") and line.strip(): indent, text = bullets[-1] bullets[-1] = (indent, f"{text} {line.strip()}") items: list[str] = [] category = "" for index, (indent, text) in enumerate(bullets): has_children = index + 1 < len(bullets) and bullets[index + 1][0] > indent if has_children: category = text.rstrip(":").strip() elif indent == 0: category = "" items.append(text) else: items.append(f"{category}: {text}" if category else text) return items def regressions(case: CaseSpec, measured: Mapping[str, float | None]) -> list[str]: """Each threshold the run exceeded, as "metric: measured > limit".""" over = [] for metric in THRESHOLD_METRICS: limit = case.thresholds.get(metric) value = measured.get(metric) if limit is not None and value is not None and value > limit: over.append(f"{metric}: {value:g} > {limit:g}") return over def _normalise(text: str) -> str: return " ".join(_NUMBERING.sub("", text.strip().lower()).split()) def _mentions(text: str, product: str) -> bool: return ( re.search(rf"(? CaseResult: deliverables: Mapping[str, Mapping[str, Any]] = interaction.state.get( "deliverables", {} ) def content(kind: DeliverableKind) -> str: item = deliverables.get(kind.value) return item["content"] if item else "" def attachments(kind: DeliverableKind) -> Mapping[str, str]: item = deliverables.get(kind.value) return item.get("attachments", {}) if item else {} checks: list[Check] = [] checks.append( Check( "completed", interaction.status == "completed", "; ".join(interaction.state.get("warnings", [])) or interaction.status, ) ) guide = content(DeliverableKind.SOLUTION_GUIDE) guide_headings = {_normalise(h) for _, h in _HEADING.findall(guide)} missing_sections = [h for h in headings if _normalise(h) not in guide_headings] checks.append( Check( "guide_follows_template", bool(guide) and not missing_sections, "missing: " + "; ".join(missing_sections) if missing_sections else "", ) ) selection = content(DeliverableKind.PRODUCT_SELECTION) selected_text = selection + "\n" + guide missing_products = [ p for p in case.required_products if not _mentions(selected_text, p) ] checks.append( Check( "required_products_selected", bool(selection) and not missing_products, "missing: " + ", ".join(missing_products) if missing_products else "", ) ) lowered = selection.lower() checks.append( Check( "selection_has_alternatives", "alternative" in lowered and "pros" in lowered and "cons" in lowered, ) ) checks.append( Check( "selection_cites_documentation", bool(_DOC_LINK.search(selection)), ) ) requirements = content(DeliverableKind.REQUIREMENTS) leaked = [ product for product in case.required_products if _mentions(requirements, product) and not _mentions(case.workflow_request, product) ] checks.append( Check( "requirements_name_no_products", bool(requirements) and not leaked, "names: " + ", ".join(leaked) if leaked else "", ) ) diagram = attachments(DeliverableKind.TARGET_ARCHITECTURE).get( "architecture.mmd", "" ) mermaid = await mermaid_check(diagram) checks.append( Check( "diagram_is_valid_mermaid", bool(diagram) and mermaid.message.startswith("Mermaid check passed"), mermaid.message, ) ) terraform_files = { name.removeprefix("terraform/"): text for name, text in attachments(DeliverableKind.DEPLOYMENT_GUIDANCE).items() if name.endswith((".tf", ".tfvars", ".tf.json")) } if terraform_files: validation = await terraform_validate(terraform_files) terraform_ok = validation.message.startswith("terraform validate passed") terraform_detail = validation.message[:1500] else: terraform_ok, terraform_detail = False, "no Terraform files attached" checks.append(Check("terraform_validates", terraform_ok, terraform_detail)) if case.code_standards: checks.append(_code_standards(case.code_standards, terraform_files)) checks.append(_chain_check(deliverables)) checks.append(_verdicts_check(attachments(DeliverableKind.VALIDATION_RESULTS))) checks.append( _diagram_matches_graph(attachments(DeliverableKind.TARGET_ARCHITECTURE)) ) if case.interactive: checks.append(_asks_before_submitting(timeline)) if judge is not None: recommendations = content(DeliverableKind.DESIGN_RECOMMENDATIONS) checks.append( await _checklist_coverage(judge, recommendations, pillar_checklists or {}) ) checks.append(await _quality(judge, recommendations)) return CaseResult(case.id, interaction.status, tuple(checks)) def _code_standards( standards: Sequence[Mapping[str, str]], files: Mapping[str, str] ) -> Check: """Whether the code holds what the case requires of it and nothing it forbids.""" code = "\n".join(files.values()) broken = [] for standard in standards: required, forbidden = standard.get("require"), standard.get("forbid") if required and not re.search(required, code): broken.append(f"{standard['name']}: not found") found = re.search(forbidden, code) if forbidden else None if found: broken.append(f"{standard['name']}: found {found.group(0)}") return Check("code_follows_standards", bool(code) and not broken, "; ".join(broken)) def _chain_check(deliverables: Mapping[str, Mapping[str, Any]]) -> Check: """The Requirements chain holds end to end: every requirement reaches a Check, directly or through a component's Element, and nothing along it is missing.""" name = "requirements_chain_complete" graph = read_graph(deliverables) if graph is None: return Check(name, False, "no well-formed Target architecture") gaps = chain_gaps( graph, requirements=read_list(deliverables, "requirements", REQUIREMENTS_FILE), components=read_list(deliverables, "technical_decomposition", COMPONENTS_FILE), products=read_list(deliverables, "product_selection", PRODUCTS_FILE), recommendations=read_list( deliverables, "design_recommendations", RECOMMENDATIONS_FILE ), ) return Check(name, not gaps, "; ".join(gaps)) def _verdicts_check(attachments: Mapping[str, str]) -> Check: """Validation ended with every Check passed and the dry-run not failed.""" name = "every_check_passes" text = attachments.get(VERDICTS_FILE) if text is None: return Check(name, False, f"no {VERDICTS_FILE}") outcome = validation_outcome(text) open_checks = [f"{item['check']} failed" for item in outcome["failed"]] + [ f"{item['check']} not verifiable" for item in outcome["not_verifiable"] ] if outcome["dry_run"].get("verdict") == "fail": open_checks.append("dry-run failed") if not outcome["passed"] and not open_checks: open_checks.append("no verdicts") return Check(name, not open_checks, "; ".join(open_checks)) def _diagram_matches_graph(attachments: Mapping[str, str]) -> Check: name = "diagram_matches_graph" try: graph = parse_graph(attachments.get(GRAPH_ATTACHMENT, "")) except GraphError as exc: return Check(name, False, exc.problems[0]) matches = attachments.get(DIAGRAM_ATTACHMENT) == render_mermaid(graph) return Check(name, matches, "" if matches else "the diagram is not the graph's") def _asks_before_submitting(timeline: Sequence[Mapping[str, Any]]) -> Check: """In an interactive Execution every Specialist confirms with the user before its first submission (spec risk D32).""" asked: set[str] = set() silent: list[str] = [] seen: set[str] = set() for run in timeline: deliverable = run.get("deliverable") if not deliverable: continue if run.get("outcome") == "NeedsInput": asked.add(deliverable) elif run.get("outcome") == "Submitted" and deliverable not in seen: seen.add(deliverable) if deliverable not in asked: silent.append(deliverable) return Check( "specialists_ask_before_submitting", bool(seen) and not silent, "submitted without asking: " + ", ".join(silent) if silent else "", ) async def _checklist_coverage( judge: Judge, recommendations: str, checklists: Mapping[str, Sequence[str]] ) -> Check: name = "pillar_checklists_covered" questions = { f"{pillar}.{index}": { "kind": "yes_no", "text": checklist_question(pillar, item), } for pillar, items in checklists.items() for index, item in enumerate(items) } if not recommendations or not questions: return Check(name, False, "no design recommendations or no checklists") try: scores = await judge.answer(recommendations, questions) except JudgeUnavailable as exc: return Check(name, False, f"judge unavailable: {exc}") per_pillar = { pillar: sum(scores.get(f"{pillar}.{i}", 0) for i in range(len(items))) / len(items) for pillar, items in checklists.items() if items } weakest = min(per_pillar.values()) detail = ", ".join(f"{pillar} {share:.0%}" for pillar, share in per_pillar.items()) return Check(name, weakest >= CHECKLIST_COVERAGE, detail) async def _quality(judge: Judge, recommendations: str) -> Check: name = "recommendations_quality" if not recommendations: return Check(name, False, "no design recommendations") try: scores = await judge.answer( recommendations, { key: {"kind": "graded", "text": text} for key, text in QUALITY_RUBRIC.items() }, ) except JudgeUnavailable as exc: return Check(name, False, f"judge unavailable: {exc}") mean = sum(scores.values()) / len(QUALITY_RUBRIC) detail = ", ".join(f"{key} {scores.get(key, 0):.2f}" for key in QUALITY_RUBRIC) return Check(name, mean >= QUALITY_BAR, f"mean {mean:.2f}: {detail}") # --- a Change request stays contained (iteration 7, step 7) ------------------------------ UNREQUESTED_SHARE = 0.1 # of the Baseline graph's items, changed without being reached def score_change( interaction: Interaction, baseline_tokens: float | None, tokens: float | None ) -> list[Check]: """Checks on a Change request against its Baseline. They read what the Execution recorded: the requirements' own account of what changed, and a comparison by id of the two versions' graphs. """ from app.workflows.changes import ADDED, CHANGED, REMOVED, change_summary state = interaction.state baseline = (state.get("baseline") or {}).get("deliverables") or {} deliverables = state.get("deliverables", {}) summary = change_summary(baseline, deliverables) requirements = summary["requirements"] checks = [ Check( "change_is_recorded", bool(baseline) and bool(requirements), ", ".join(f"{item['id']} {item['change']}" for item in requirements) or "no requirement is marked added, changed or removed", ) ] graph = _graph_data(deliverables) touched = set(summary["checks"][ADDED]) | set(summary["checks"][CHANGED]) removed_checks = set(summary["checks"][REMOVED]) old_sources = { item.get("id"): item.get("source") for item in _graph_data(baseline)["checks"] } serving: dict[str, set[str]] = {} for component in read_list( deliverables, "technical_decomposition", COMPONENTS_FILE ): for requirement in component.get("requirements") or (): serving.setdefault(str(requirement), set()).add(str(component.get("id"))) unreached = [] for item in requirements: identifier = item["id"] if item["change"] == REMOVED: # What it asked for should go: a Check that came from it is removed. reaches = any(old_sources.get(c) == identifier for c in removed_checks) reaches = reaches or identifier not in old_sources.values() else: # A constraint is the source of its Checks. A functional requirement is # carried by the components that serve it, so it is reached through a # Check on an Element that realises one of them. realising = { element.get("id") for element in graph["elements"] if element.get("component") in serving.get(identifier, ()) } reaches = any( check.get("id") in touched and ( check.get("source") == identifier or check.get("applies_to") in realising ) for check in graph["checks"] ) if not reaches: unreached.append(identifier) checks.append( Check( "changed_requirements_reach_checks", bool(requirements) and not unreached, "no Check changed for: " + ", ".join(unreached) if unreached else "", ) ) unrequested = [ f"{part} {identifier}" for part, ids in summary["unrequested"].items() for identifier in ids ] size = sum(len(items) for items in _graph_data(baseline).values()) or 1 checks.append( Check( "change_is_contained", len(unrequested) <= max(2, UNREQUESTED_SHARE * size), f"{len(unrequested)} of {size} items changed without being reached" + (": " + ", ".join(unrequested[:12]) if unrequested else ""), ) ) cheaper = ( baseline_tokens is not None and tokens is not None and tokens < baseline_tokens ) checks.append( Check( "change_costs_less_than_baseline", cheaper, f"{tokens or 0:,.0f} tokens against {baseline_tokens or 0:,.0f}", ) ) return checks def _graph_data( deliverables: Mapping[str, Mapping[str, Any]], ) -> dict[str, list[dict[str, Any]]]: text = ( (deliverables.get(DeliverableKind.TARGET_ARCHITECTURE.value) or {}) .get("attachments", {}) .get("target-architecture.json", "") ) try: data = json.loads(text) except ValueError: data = {} if not isinstance(data, dict): data = {} return { part: [item for item in data.get(part, []) if isinstance(item, dict)] for part in ("elements", "relationships", "checks") }