Files
gcp_solution_architecture_a…/app/tools/pillars.py
Jonathan Boniface 654370bb46 feat: all four evaluation cases pass; intake, checklist review, thresholds
Case 04 (interactive)
- A scripted answer may name who it is for, so facts written for
  requirements discovery are not given to a Specialist confirming a draft.
- A reply that restates what the agreed requirements already say is read
  as an answer, not a request for changes.
- With a user present, a first submission is refused until the user has
  been shown a draft of that Deliverable (CONFIRM_DRAFTS).

Routing and reviews
- Intake: a message that starts something is read first. A question is
  answered and starts nothing; a change to delivered work starts as a new
  request with a note, until Change requests are built.
- Requirements and recommendations list what infrastructure code can show
  (conditions) apart from what it cannot (outside_code); the latter needs
  no Check and validation proposes nothing for it.
- A requirements review, "names no products", behind its own switch (off).
- Each pillar advisor reviews its pillar's checklist; the engine writes the
  reviews into the design recommendations. The judged checklist check went
  from 9-45% a pillar to 100% in one run of case 04.

Evaluation
- Per-case thresholds for Revisions, seconds and tokens, set from every
  completed run on this build.
- A progress file after every agent run, so an interrupted run keeps its
  time and tokens.
- eval/calibrate_judge.py with a 32-question set awaiting a person's
  labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91.

Smaller
- An Execution that ends with warnings writes its Deliverables, with the
  warnings and each status in the manifest.
- CI installs the application's dependencies before linting and testing.
- A load-balancer pattern in the IaC patterns Skill.

Plans record nine runs of 2026-10-08. Iteration 6 is complete.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 09:32:57 +01:00

250 lines
9.1 KiB
Python

"""The design advisor's fan-out to one advisor per architecture-framework pillar.
Pillar advice is working material for the design recommendations Deliverable, not a
Deliverable itself, so it never touches the Ledger. It is kept in that Deliverable's
working memory, so a Revision re-consults only the pillars its feedback touches.
Each advisor also returns its review of the pillar's own checklist: for every item,
whether its advice addresses it, or why it does not apply. The reviews are kept
beside the advice and written into the Deliverable by code when it is submitted, so
an architect sees the whole checklist accounted for and no model spends output
copying it.
"""
from __future__ import annotations
import asyncio
import dataclasses
from collections.abc import Mapping, MutableMapping, Sequence
from contextlib import AsyncExitStack
from typing import Any
from app.adk.runtime import (
AgentRuntime,
AgentTask,
RoleSpec,
RunLimiter,
Submitted,
ToolResult,
ToolSpec,
)
CONSULT_TOOL = "consult_pillar_advisors"
NO_GUIDANCE = "No guidance"
ADVICE_KEY = "pillar_advice" # working memory: pillar -> the advice last given
CHECKLIST_KEY = "pillar_checklists" # working memory: pillar -> its checklist review
REVIEW_HEADING = "Architecture framework checklist review"
STATUSES = ("addressed", "not_applicable", "not_addressed")
_STATUS_WORDS = {
"addressed": "Addressed",
"not_applicable": "Does not apply",
"not_addressed": "Not addressed",
}
def pillar_consultation_tool(
runtime: AgentRuntime,
advisor: RoleSpec,
pillar_skills: Mapping[str, str],
extra_tools: Sequence[ToolSpec] = (),
memory: MutableMapping[str, Any] | None = None,
limiter: RunLimiter | None = None,
) -> ToolSpec:
"""A tool that runs one pillar advisor per pillar, in parallel, each with its Skill.
With `memory`, advice is remembered: naming `pillars` re-consults only those and
returns the remembered advice for the rest.
"""
async def advise(pillar: str, brief: str) -> tuple[str, list[dict[str, str]]]:
async with AsyncExitStack() as stack:
if limiter:
await stack.enter_async_context(limiter.slot())
return await _advise(
runtime, advisor, pillar, pillar_skills[pillar], brief, extra_tools
)
async def consult(brief: str, pillars: Sequence[str] | None = None) -> ToolResult:
chosen = list(pillars) if pillars else list(pillar_skills)
unknown = [pillar for pillar in chosen if pillar not in pillar_skills]
if unknown:
return ToolResult(
f"Refused: unknown pillars {', '.join(unknown)}; "
f"choose from {', '.join(pillar_skills)}"
)
async with AsyncExitStack() as stack:
if limiter:
# The calling Specialist only waits here, so its slot goes to advisors.
await stack.enter_async_context(limiter.lend())
fresh = dict(
zip(
chosen,
await asyncio.gather(*(advise(pillar, brief) for pillar in chosen)),
)
)
remembered: dict[str, str] = (
dict(memory.get(ADVICE_KEY, {})) if memory is not None else {}
)
reviews: dict[str, list[dict[str, str]]] = (
dict(memory.get(CHECKLIST_KEY, {})) if memory is not None else {}
)
sections = []
for pillar in pillar_skills:
if pillar in fresh:
advice, review = fresh[pillar]
if not advice.startswith(NO_GUIDANCE):
remembered[pillar] = advice
reviews[pillar] = review
sections.append(f"## {pillar}\n\n{advice}")
elif pillar in remembered:
sections.append(
f"## {pillar} (from your earlier consultation)\n\n"
f"{remembered[pillar]}"
)
if memory is not None:
memory[ADVICE_KEY] = remembered
memory[CHECKLIST_KEY] = reviews
return ToolResult("\n\n".join(sections))
return ToolSpec(
CONSULT_TOOL,
"Get guidance from one advisor per architecture-framework pillar, in parallel. "
"Give a brief with the workload, its requirements and the selected products. "
"Omit `pillars` to consult every pillar afresh. Name `pillars` to re-consult "
"only those; advice already given for the others is returned unchanged.",
{
"type": "object",
"properties": {
"brief": {"type": "string"},
"pillars": {
"type": "array",
"items": {"type": "string", "enum": list(pillar_skills)},
},
},
"required": ["brief"],
},
consult,
)
async def _advise(
runtime: AgentRuntime,
advisor: RoleSpec,
pillar: str,
skill: str,
brief: str,
extra_tools: Sequence[ToolSpec],
) -> tuple[str, list[dict[str, str]]]:
"""One advisor's advice, and its review of the pillar's checklist."""
role = dataclasses.replace(advisor, skills=(skill,))
task = AgentTask(
role=advisor.name,
prompt=f"Advise on the {pillar} pillar for this workload.",
context={"pillar": pillar, "brief": brief},
)
review: list[dict[str, str]] = []
try:
outcome = await runtime.run(
role, task, [_submit_advice_tool(review), *extra_tools]
)
except Exception as exc: # noqa: BLE001 - one failed advisor must not lose the others
return f"{NO_GUIDANCE}: the advisor failed ({exc}).", []
if isinstance(outcome, Submitted):
return outcome.content.strip(), review
return f"{NO_GUIDANCE}: the advisor ended with {type(outcome).__name__}.", []
def _submit_advice_tool(review: list[dict[str, str]]) -> ToolSpec:
"""`review` receives the checklist review the advisor submits with its advice."""
async def submit_advice(content: str, checklist: Sequence[Any] = ()) -> ToolResult:
if not content.strip():
return ToolResult("Not submitted: the advice is empty.")
review[:] = [
{
"item": " ".join(str(entry.get("item", "")).split()),
"status": str(entry.get("status", "")),
"note": " ".join(str(entry.get("note", "")).split()),
}
for entry in checklist or ()
if isinstance(entry, Mapping)
and str(entry.get("item", "")).strip()
and entry.get("status") in STATUSES
]
return ToolResult("Advice submitted.", Submitted(content))
text = {"type": "string"}
return ToolSpec(
"submit_advice",
"Submit your pillar recommendations as markdown, with your review of the "
"pillar's checklist. Ends your turn.",
{
"type": "object",
"properties": {
"content": text,
"checklist": {
"type": "array",
"description": "One entry for every item of the Skill's "
"validation checklist.",
"items": {
"type": "object",
"properties": {
"item": text,
"status": {"type": "string", "enum": list(STATUSES)},
"note": {
"type": "string",
"description": "How your advice addresses it, or "
"why it does not apply to this workload.",
},
},
"required": ["item", "status", "note"],
},
},
},
"required": ["content"],
},
submit_advice,
tier="execution",
)
def render_checklist_review(working: Mapping[str, Any]) -> str:
"""The advisors' checklist reviews as one section of markdown tables, or "" when
there are none. Written by code from what the advisors recorded."""
reviews = working.get(CHECKLIST_KEY)
if not isinstance(reviews, Mapping) or not any(reviews.values()):
return ""
parts = [
f"## {REVIEW_HEADING}",
"",
(
"Each pillar advisor's review of its pillar's checklist for this "
"workload: what its advice addresses, and what does not apply and why."
),
]
for pillar, entries in reviews.items():
if not entries:
continue
parts += [
"",
f"### {pillar}",
"",
"| Checklist item | Review | How, or why not |",
"| --- | --- | --- |",
]
parts += [
"| "
+ " | ".join(
str(value).replace("|", "\\|")
for value in (
entry.get("item", ""),
_STATUS_WORDS.get(str(entry.get("status")), ""),
entry.get("note", ""),
)
)
+ " |"
for entry in entries
if isinstance(entry, Mapping)
]
return "\n".join(parts) + "\n"