Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
250 lines
9.1 KiB
Python
250 lines
9.1 KiB
Python
"""The design advisor's fan-out to one advisor per architecture-framework pillar.
|
|
|
|
Pillar advice is working material for the design recommendations Deliverable, not a
|
|
Deliverable itself, so it never touches the Ledger. It is kept in that Deliverable's
|
|
working memory, so a Revision re-consults only the pillars its feedback touches.
|
|
|
|
Each advisor also returns its review of the pillar's own checklist: for every item,
|
|
whether its advice addresses it, or why it does not apply. The reviews are kept
|
|
beside the advice and written into the Deliverable by code when it is submitted, so
|
|
an architect sees the whole checklist accounted for and no model spends output
|
|
copying it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import dataclasses
|
|
from collections.abc import Mapping, MutableMapping, Sequence
|
|
from contextlib import AsyncExitStack
|
|
from typing import Any
|
|
|
|
from app.adk.runtime import (
|
|
AgentRuntime,
|
|
AgentTask,
|
|
RoleSpec,
|
|
RunLimiter,
|
|
Submitted,
|
|
ToolResult,
|
|
ToolSpec,
|
|
)
|
|
|
|
CONSULT_TOOL = "consult_pillar_advisors"
|
|
NO_GUIDANCE = "No guidance"
|
|
ADVICE_KEY = "pillar_advice" # working memory: pillar -> the advice last given
|
|
CHECKLIST_KEY = "pillar_checklists" # working memory: pillar -> its checklist review
|
|
REVIEW_HEADING = "Architecture framework checklist review"
|
|
STATUSES = ("addressed", "not_applicable", "not_addressed")
|
|
_STATUS_WORDS = {
|
|
"addressed": "Addressed",
|
|
"not_applicable": "Does not apply",
|
|
"not_addressed": "Not addressed",
|
|
}
|
|
|
|
|
|
def pillar_consultation_tool(
|
|
runtime: AgentRuntime,
|
|
advisor: RoleSpec,
|
|
pillar_skills: Mapping[str, str],
|
|
extra_tools: Sequence[ToolSpec] = (),
|
|
memory: MutableMapping[str, Any] | None = None,
|
|
limiter: RunLimiter | None = None,
|
|
) -> ToolSpec:
|
|
"""A tool that runs one pillar advisor per pillar, in parallel, each with its Skill.
|
|
|
|
With `memory`, advice is remembered: naming `pillars` re-consults only those and
|
|
returns the remembered advice for the rest.
|
|
"""
|
|
|
|
async def advise(pillar: str, brief: str) -> tuple[str, list[dict[str, str]]]:
|
|
async with AsyncExitStack() as stack:
|
|
if limiter:
|
|
await stack.enter_async_context(limiter.slot())
|
|
return await _advise(
|
|
runtime, advisor, pillar, pillar_skills[pillar], brief, extra_tools
|
|
)
|
|
|
|
async def consult(brief: str, pillars: Sequence[str] | None = None) -> ToolResult:
|
|
chosen = list(pillars) if pillars else list(pillar_skills)
|
|
unknown = [pillar for pillar in chosen if pillar not in pillar_skills]
|
|
if unknown:
|
|
return ToolResult(
|
|
f"Refused: unknown pillars {', '.join(unknown)}; "
|
|
f"choose from {', '.join(pillar_skills)}"
|
|
)
|
|
async with AsyncExitStack() as stack:
|
|
if limiter:
|
|
# The calling Specialist only waits here, so its slot goes to advisors.
|
|
await stack.enter_async_context(limiter.lend())
|
|
fresh = dict(
|
|
zip(
|
|
chosen,
|
|
await asyncio.gather(*(advise(pillar, brief) for pillar in chosen)),
|
|
)
|
|
)
|
|
remembered: dict[str, str] = (
|
|
dict(memory.get(ADVICE_KEY, {})) if memory is not None else {}
|
|
)
|
|
reviews: dict[str, list[dict[str, str]]] = (
|
|
dict(memory.get(CHECKLIST_KEY, {})) if memory is not None else {}
|
|
)
|
|
sections = []
|
|
for pillar in pillar_skills:
|
|
if pillar in fresh:
|
|
advice, review = fresh[pillar]
|
|
if not advice.startswith(NO_GUIDANCE):
|
|
remembered[pillar] = advice
|
|
reviews[pillar] = review
|
|
sections.append(f"## {pillar}\n\n{advice}")
|
|
elif pillar in remembered:
|
|
sections.append(
|
|
f"## {pillar} (from your earlier consultation)\n\n"
|
|
f"{remembered[pillar]}"
|
|
)
|
|
if memory is not None:
|
|
memory[ADVICE_KEY] = remembered
|
|
memory[CHECKLIST_KEY] = reviews
|
|
return ToolResult("\n\n".join(sections))
|
|
|
|
return ToolSpec(
|
|
CONSULT_TOOL,
|
|
"Get guidance from one advisor per architecture-framework pillar, in parallel. "
|
|
"Give a brief with the workload, its requirements and the selected products. "
|
|
"Omit `pillars` to consult every pillar afresh. Name `pillars` to re-consult "
|
|
"only those; advice already given for the others is returned unchanged.",
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"brief": {"type": "string"},
|
|
"pillars": {
|
|
"type": "array",
|
|
"items": {"type": "string", "enum": list(pillar_skills)},
|
|
},
|
|
},
|
|
"required": ["brief"],
|
|
},
|
|
consult,
|
|
)
|
|
|
|
|
|
async def _advise(
|
|
runtime: AgentRuntime,
|
|
advisor: RoleSpec,
|
|
pillar: str,
|
|
skill: str,
|
|
brief: str,
|
|
extra_tools: Sequence[ToolSpec],
|
|
) -> tuple[str, list[dict[str, str]]]:
|
|
"""One advisor's advice, and its review of the pillar's checklist."""
|
|
role = dataclasses.replace(advisor, skills=(skill,))
|
|
task = AgentTask(
|
|
role=advisor.name,
|
|
prompt=f"Advise on the {pillar} pillar for this workload.",
|
|
context={"pillar": pillar, "brief": brief},
|
|
)
|
|
review: list[dict[str, str]] = []
|
|
try:
|
|
outcome = await runtime.run(
|
|
role, task, [_submit_advice_tool(review), *extra_tools]
|
|
)
|
|
except Exception as exc: # noqa: BLE001 - one failed advisor must not lose the others
|
|
return f"{NO_GUIDANCE}: the advisor failed ({exc}).", []
|
|
if isinstance(outcome, Submitted):
|
|
return outcome.content.strip(), review
|
|
return f"{NO_GUIDANCE}: the advisor ended with {type(outcome).__name__}.", []
|
|
|
|
|
|
def _submit_advice_tool(review: list[dict[str, str]]) -> ToolSpec:
|
|
"""`review` receives the checklist review the advisor submits with its advice."""
|
|
|
|
async def submit_advice(content: str, checklist: Sequence[Any] = ()) -> ToolResult:
|
|
if not content.strip():
|
|
return ToolResult("Not submitted: the advice is empty.")
|
|
review[:] = [
|
|
{
|
|
"item": " ".join(str(entry.get("item", "")).split()),
|
|
"status": str(entry.get("status", "")),
|
|
"note": " ".join(str(entry.get("note", "")).split()),
|
|
}
|
|
for entry in checklist or ()
|
|
if isinstance(entry, Mapping)
|
|
and str(entry.get("item", "")).strip()
|
|
and entry.get("status") in STATUSES
|
|
]
|
|
return ToolResult("Advice submitted.", Submitted(content))
|
|
|
|
text = {"type": "string"}
|
|
return ToolSpec(
|
|
"submit_advice",
|
|
"Submit your pillar recommendations as markdown, with your review of the "
|
|
"pillar's checklist. Ends your turn.",
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"content": text,
|
|
"checklist": {
|
|
"type": "array",
|
|
"description": "One entry for every item of the Skill's "
|
|
"validation checklist.",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"item": text,
|
|
"status": {"type": "string", "enum": list(STATUSES)},
|
|
"note": {
|
|
"type": "string",
|
|
"description": "How your advice addresses it, or "
|
|
"why it does not apply to this workload.",
|
|
},
|
|
},
|
|
"required": ["item", "status", "note"],
|
|
},
|
|
},
|
|
},
|
|
"required": ["content"],
|
|
},
|
|
submit_advice,
|
|
tier="execution",
|
|
)
|
|
|
|
|
|
def render_checklist_review(working: Mapping[str, Any]) -> str:
|
|
"""The advisors' checklist reviews as one section of markdown tables, or "" when
|
|
there are none. Written by code from what the advisors recorded."""
|
|
reviews = working.get(CHECKLIST_KEY)
|
|
if not isinstance(reviews, Mapping) or not any(reviews.values()):
|
|
return ""
|
|
parts = [
|
|
f"## {REVIEW_HEADING}",
|
|
"",
|
|
(
|
|
"Each pillar advisor's review of its pillar's checklist for this "
|
|
"workload: what its advice addresses, and what does not apply and why."
|
|
),
|
|
]
|
|
for pillar, entries in reviews.items():
|
|
if not entries:
|
|
continue
|
|
parts += [
|
|
"",
|
|
f"### {pillar}",
|
|
"",
|
|
"| Checklist item | Review | How, or why not |",
|
|
"| --- | --- | --- |",
|
|
]
|
|
parts += [
|
|
"| "
|
|
+ " | ".join(
|
|
str(value).replace("|", "\\|")
|
|
for value in (
|
|
entry.get("item", ""),
|
|
_STATUS_WORDS.get(str(entry.get("status")), ""),
|
|
entry.get("note", ""),
|
|
)
|
|
)
|
|
+ " |"
|
|
for entry in entries
|
|
if isinstance(entry, Mapping)
|
|
]
|
|
return "\n".join(parts) + "\n"
|