"""Every Execution judged as it ends (ADR-0007, iteration 5 step 19). The evaluation scores benchmark cases. This asks the same graded questions of a live Execution's design recommendations when it ends, and keeps the answers with its record. It reports; it never stops or changes an Execution. Without a decision an Execution carries no assessment. """ from __future__ import annotations import logging from collections.abc import Mapping from typing import Any from app.adk.runtime import Decider, Question logger = logging.getLogger(__name__) GRADES = ("not at all", "weakly", "mostly", "fully") QUALITY_BAR = 0.67 # mean of the graded rubric: "mostly" or better QUALITY_RUBRIC: Mapping[str, str] = { "specific": "Are the recommendations specific to this workload and its stated " "requirements, as opposed to generic best practice that would fit any workload?", "actionable": "Does each recommendation name the concrete setting, feature or " "configuration to apply, so an engineer could act on it without more research?", "trade_offs": "Where pillars pull in different directions (for example cost " "against reliability), is the trade-off named and a resolution recommended?", "traceable": "Can each recommendation be traced to a requirement or constraint " "of the workload?", } _JUDGED = "design_recommendations" def checklist_question(pillar: str, item: str) -> str: """Whether the recommendations account for one item of a pillar's checklist. An item is accounted for in three ways. The third matters: a customer's request seldom says enough to settle every item, and what it leaves unsaid is a question for them, not a fault in the design (the owner, 2026-10-10). """ return ( f"Do the recommendations account for this {pillar} checklist item in any " "of these ways: by recommending it, by explaining why it does not apply, " "or by recording it as something to confirm with the customer? " f"{item}" ) async def assess( decider: Decider, deliverables: Mapping[str, Mapping[str, Any]], *, scope: str = "", ) -> dict[str, Any] | None: """The rubric's scores for the design recommendations, each from 0 to 1, with their mean and whether it meets the bar. None when there is nothing to judge or no decision.""" document = str((deliverables.get(_JUDGED) or {}).get("content", "")) if not document.strip(): return None answers = await decider.decide( document, {key: Question("score", text, GRADES) for key, text in QUALITY_RUBRIC.items()}, scope=scope, ) if answers is None: return None top = len(GRADES) - 1 scores = { key: round(min(1.0, max(0.0, float(answers[key].value) / top)), 2) for key in QUALITY_RUBRIC } mean = round(sum(scores.values()) / len(scores), 2) if mean < QUALITY_BAR: logger.warning("[%s] design recommendations judged at %.2f", scope, mean) return { "recommendations_quality": scores, "mean": mean, "meets_bar": mean >= QUALITY_BAR, }