The reference agent's subject, cloud solution architecture, as an agent in the operating model agent's pattern: the Execution engine (from orchestration_engine) and one Workflow in app/solution_architecture, built for the settings and provider bundle by assembly.workflow_for. Copied from gcp_solution_architecture_agent at 8b397cd without its own engine: its role files, Skills, provider bundle, ADRs, vocabulary, scripts and evaluation datasets. The image takes the engine from a named build context and installs Terraform. It is called through the Orchestration engine, and its files are what the operating model agent digests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
82 lines
3.1 KiB
Python
82 lines
3.1 KiB
Python
"""Every Execution judged as it ends (ADR-0007, iteration 5 step 19).
|
|
|
|
The evaluation scores benchmark cases. This asks the same graded questions of a live
|
|
Execution's design recommendations when it ends, and keeps the answers with its
|
|
record. It reports; it never stops or changes an Execution.
|
|
|
|
Without a decision an Execution carries no assessment.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import Mapping
|
|
from typing import Any
|
|
|
|
from execution_engine.adk.runtime import Decider, Question
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
GRADES = ("not at all", "weakly", "mostly", "fully")
|
|
QUALITY_BAR = 0.67 # mean of the graded rubric: "mostly" or better
|
|
QUALITY_RUBRIC: Mapping[str, str] = {
|
|
"specific": "Are the recommendations specific to this workload and its stated "
|
|
"requirements, as opposed to generic best practice that would fit any workload?",
|
|
"actionable": "Does each recommendation name the concrete setting, feature or "
|
|
"configuration to apply, so an engineer could act on it without more research?",
|
|
"trade_offs": "Where pillars pull in different directions (for example cost "
|
|
"against reliability), is the trade-off named and a resolution recommended?",
|
|
"traceable": "Can each recommendation be traced to a requirement or constraint "
|
|
"of the workload?",
|
|
}
|
|
_JUDGED = "design_recommendations"
|
|
|
|
|
|
def checklist_question(pillar: str, item: str) -> str:
|
|
"""Whether the recommendations account for one item of a pillar's checklist.
|
|
|
|
An item is accounted for in three ways. The third matters: a customer's request
|
|
seldom says enough to settle every item, and what it leaves unsaid is a
|
|
question for them, not a fault in the design (the owner, 2026-10-10).
|
|
"""
|
|
return (
|
|
f"Do the recommendations account for this {pillar} checklist item in any "
|
|
"of these ways: by recommending it, by explaining why it does not apply, "
|
|
"or by recording it as something to confirm with the customer? "
|
|
f"{item}"
|
|
)
|
|
|
|
|
|
async def assess(
|
|
decider: Decider,
|
|
deliverables: Mapping[str, Mapping[str, Any]],
|
|
*,
|
|
scope: str = "",
|
|
) -> dict[str, Any] | None:
|
|
"""The rubric's scores for the design recommendations, each from 0 to 1, with
|
|
their mean and whether it meets the bar. None when there is nothing to judge or
|
|
no decision."""
|
|
document = str((deliverables.get(_JUDGED) or {}).get("content", ""))
|
|
if not document.strip():
|
|
return None
|
|
answers = await decider.decide(
|
|
document,
|
|
{key: Question("score", text, GRADES) for key, text in QUALITY_RUBRIC.items()},
|
|
scope=scope,
|
|
)
|
|
if answers is None:
|
|
return None
|
|
top = len(GRADES) - 1
|
|
scores = {
|
|
key: round(min(1.0, max(0.0, float(answers[key].value) / top)), 2)
|
|
for key in QUALITY_RUBRIC
|
|
}
|
|
mean = round(sum(scores.values()) / len(scores), 2)
|
|
if mean < QUALITY_BAR:
|
|
logger.warning("[%s] design recommendations judged at %.2f", scope, mean)
|
|
return {
|
|
"recommendations_quality": scores,
|
|
"mean": mean,
|
|
"meets_bar": mean >= QUALITY_BAR,
|
|
}
|