feat: all four evaluation cases pass; intake, checklist review, thresholds

Case 04 (interactive)
- A scripted answer may name who it is for, so facts written for
  requirements discovery are not given to a Specialist confirming a draft.
- A reply that restates what the agreed requirements already say is read
  as an answer, not a request for changes.
- With a user present, a first submission is refused until the user has
  been shown a draft of that Deliverable (CONFIRM_DRAFTS).

Routing and reviews
- Intake: a message that starts something is read first. A question is
  answered and starts nothing; a change to delivered work starts as a new
  request with a note, until Change requests are built.
- Requirements and recommendations list what infrastructure code can show
  (conditions) apart from what it cannot (outside_code); the latter needs
  no Check and validation proposes nothing for it.
- A requirements review, "names no products", behind its own switch (off).
- Each pillar advisor reviews its pillar's checklist; the engine writes the
  reviews into the design recommendations. The judged checklist check went
  from 9-45% a pillar to 100% in one run of case 04.

Evaluation
- Per-case thresholds for Revisions, seconds and tokens, set from every
  completed run on this build.
- A progress file after every agent run, so an interrupted run keeps its
  time and tokens.
- eval/calibrate_judge.py with a 32-question set awaiting a person's
  labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91.

Smaller
- An Execution that ends with warnings writes its Deliverables, with the
  warnings and each status in the manifest.
- CI installs the application's dependencies before linting and testing.
- A load-balancer pattern in the IaC patterns Skill.

Plans record nine runs of 2026-10-08. Iteration 6 is complete.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 09:32:57 +01:00
parent d41607f505
commit 654370bb46
53 changed files with 2122 additions and 106 deletions

View File

@@ -3,6 +3,10 @@
A new task starts an Execution; a message on an input-required task answers the
Execution's prompt. Approvals and permissions count only as structured data parts,
never as free text.
Before a new task starts anything its message is read (app/nodes/intake.py): a
question is answered and starts nothing, and a change to delivered work is started
as a new request with a note saying so, until Change requests are built.
"""
from __future__ import annotations
@@ -25,6 +29,7 @@ from google.protobuf import json_format
from app.agent import ExecutionEngine, Interaction
from app.nodes.execution_nodes import GUIDE_FILENAME
from app.nodes.intake import ABOUT, CHANGE_NOTE, CHANGE_REQUEST, QUESTION, Intake
from app.presentation import describe_prompt
from app.workflows.deliverables import DeliverableKind
@@ -34,14 +39,17 @@ STRUCTURED_REPLY_KEYS = ("approval", "permission")
class SolutionArchitectExecutor(AgentExecutor):
def __init__(self, engine: ExecutionEngine) -> None:
def __init__(self, engine: ExecutionEngine, intake: Intake | None = None) -> None:
self._engine = engine
self._intake = intake
async def execute(self, context: RequestContext, event_queue: EventQueue) -> None:
task_id = context.task_id or ""
updater = TaskUpdater(event_queue, task_id, context.context_id or "")
new = context.current_task is None
note: str | None = None
try:
if context.current_task is None:
if new:
history = [context.message] if context.message is not None else []
await event_queue.enqueue_event(
new_task(
@@ -53,6 +61,27 @@ class SolutionArchitectExecutor(AgentExecutor):
)
await updater.start_work()
options = _options(context)
# A pre-approved caller is a batch job: it sends work, not questions.
if self._intake and not options.get("pre_approved"):
reading = await self._intake.read(
context.get_user_input(),
options.get("customer_case"),
scope=task_id,
)
if reading.kind == QUESTION:
await updater.complete(
updater.new_agent_message(
[
new_text_part(ABOUT),
new_data_part(
{"status": "answered", "intake": QUESTION}
),
]
)
)
return
if reading.kind == CHANGE_REQUEST:
note = CHANGE_NOTE
interaction = await self._engine.start(
context.get_user_input(),
session_id=context.context_id,
@@ -76,7 +105,7 @@ class SolutionArchitectExecutor(AgentExecutor):
)
)
return
await self._publish(interaction, updater)
await self._publish(interaction, updater, note if new else None)
async def cancel(self, context: RequestContext, event_queue: EventQueue) -> None:
# The checkpoint stays; cancelling only ends the A2A task.
@@ -85,12 +114,15 @@ class SolutionArchitectExecutor(AgentExecutor):
)
await updater.cancel()
async def _publish(self, interaction: Interaction, updater: TaskUpdater) -> None:
async def _publish(
self, interaction: Interaction, updater: TaskUpdater, note: str | None = None
) -> None:
if interaction.status == "awaiting_user" and interaction.prompt is not None:
prompt = dict(interaction.prompt)
await updater.requires_input(
updater.new_agent_message(
[
*([new_text_part(note)] if note else []),
new_text_part(describe_prompt(prompt)),
new_data_part({"prompt": prompt}),
]

View File

@@ -35,6 +35,7 @@ from app.adk.runtime import (
from app.checkpointing import open_checkpointer
from app.config import Settings
from app.customer_case import record_intent
from app.nodes.intake import Intake
from app.nodes.router import (
REPLY_WRITER,
OrchestratorPolicy,
@@ -52,6 +53,7 @@ from app.tools.execution import ORCHESTRATOR
from app.tools.jev import JevDecider, JevLimits
from app.tools.local import TerraformRunner, local_tools
from app.tools.mcp import McpCatalog, auth_from_config
from app.tools.requirements_review import RequirementsReview
from app.tools.schema_review import Reviews, SchemaReview
from app.tools.toolkit import make_tool_factory
from app.workflows.deliverables import DEPENDENCY_GRAPH
@@ -88,6 +90,7 @@ class ExecutionEngine:
skill_versions: Mapping[str, str] | None = None,
orchestrator: OrchestratorPolicy | None = None,
submission_review: SubmissionReview | None = None,
confirm_drafts: bool = False,
) -> None:
required = {
ORCHESTRATOR,
@@ -109,6 +112,7 @@ class ExecutionEngine:
skill_versions=skill_versions,
orchestrator=orchestrator,
submission_review=submission_review,
confirm_drafts=confirm_drafts,
).compile(checkpointer=checkpointer)
async def start(
@@ -120,9 +124,9 @@ class ExecutionEngine:
pre_approved: bool = False,
allow_cloud_read: bool | None = None,
execution_id: str | None = None,
scripted_answers: Sequence[tuple[str, str]] = (),
scripted_answers: Sequence[Sequence[str]] = (),
) -> Interaction:
"""Start an Execution. `scripted_answers` are (regex, answer) pairs a
"""Start an Execution. `scripted_answers` are (regex, answer[, asker]) entries a
pre-approved Execution uses to answer questions, for evaluation runs."""
customer_case = check_customer_case(customer_case)
execution_id = execution_id or str(uuid.uuid4())
@@ -156,6 +160,7 @@ class ExecutionEngine:
"awaiting_questions": [],
"answers": {},
"drafts": {},
"drafts_shown": [],
"working": {},
"pending_permission": None,
"prompt": None,
@@ -175,9 +180,7 @@ class ExecutionEngine:
"turns": 0,
"status": "running",
"warnings": [],
"scripted_answers": [
[pattern, answer] for pattern, answer in scripted_answers
],
"scripted_answers": [list(entry) for entry in scripted_answers],
}
await self._graph.ainvoke(initial, self._config(execution_id))
return await self.interaction(execution_id)
@@ -236,6 +239,7 @@ class Assembly:
roles: dict[str, RoleSpec]
mcp_gaps: dict[str, str]
decider: Decider = field(default_factory=NoDecider)
intake: Intake | None = None # reads a message that starts something
def open_decider(settings: Settings) -> Decider:
@@ -334,6 +338,11 @@ async def open_assembly(
if bundle.terraform_providers:
# In code: every setting a Check names exists on its resource type.
reviews.append(SchemaReview(runner))
if settings.JEV_REQUIREMENTS_REVIEW and isinstance(decider, JevDecider):
# Optional, and off by default: Jev says whether a requirement names a product.
reviews.append(
RequirementsReview(decider, act=settings.JEV_REQUIREMENTS_THRESHOLD)
)
if settings.JEV_COVERAGE_REVIEW and isinstance(decider, JevDecider):
# Optional, and off by default: Jev says which Check verifies each condition.
reviews.append(CoverageReview(decider, none_at=settings.JEV_COVERAGE_THRESHOLD))
@@ -354,9 +363,21 @@ async def open_assembly(
},
orchestrator=orchestrator,
submission_review=Reviews(reviews) if reviews else None,
confirm_drafts=settings.CONFIRM_DRAFTS,
)
try:
yield Assembly(engine, bundle, roles, dict(catalog.gaps), decider)
yield Assembly(
engine,
bundle,
roles,
dict(catalog.gaps),
decider,
Intake(
decider,
output_root or settings.BASE_DIR,
act=settings.JEV_ACT_THRESHOLD,
),
)
finally:
if isinstance(decider, JevDecider):
await decider.aclose()

View File

@@ -54,6 +54,9 @@ class Settings(BaseSettings):
A2A_PUBLIC_URL: str | None = None # advertised in the agent card
ORCHESTRATOR_BUDGET: int = 40
SPECIALIST_BUDGET: int = 15
# With a user present, a Specialist's first submission of a Deliverable is refused
# until it has put its draft to them. Off: Specialists are only told to.
CONFIRM_DRAFTS: bool = True
MAX_REVISIONS: int = 3
MAX_ORCHESTRATOR_TURNS: int = 100
# How an Orchestrator turn is decided (ADR-0007): "router" reads the Execution
@@ -87,6 +90,10 @@ class Settings(BaseSettings):
JEV_COVERAGE_REVIEW: bool = False
# How likely "no Check verifies this" must be before the condition is raised.
JEV_COVERAGE_THRESHOLD: float = 0.6
# Review the requirements at submission: does any name a cloud product? Off
# until a full case has run with it.
JEV_REQUIREMENTS_REVIEW: bool = False
JEV_REQUIREMENTS_THRESHOLD: float = 0.3
# Paths
BASE_DIR: Path = Path(__file__).resolve().parent.parent

View File

@@ -85,7 +85,9 @@ def create_app(
async def lifespan(app: Starlette) -> AsyncIterator[None]:
async with assembly_factory(settings) as assembly:
handler = DefaultRequestHandlerV2(
agent_executor=SolutionArchitectExecutor(assembly.engine),
agent_executor=SolutionArchitectExecutor(
assembly.engine, assembly.intake
),
task_store=InMemoryTaskStore(),
agent_card=card,
)

View File

@@ -105,6 +105,7 @@ def make_execution_nodes(
skill_versions: Mapping[str, str] | None = None,
orchestrator: OrchestratorPolicy | None = None,
submission_review: SubmissionReview | None = None,
confirm_drafts: bool = False,
) -> ExecutionNodes:
tool_factory = tool_factory or _no_tools
# How an Orchestrator turn is decided. The default drives the tools from the
@@ -180,6 +181,7 @@ def make_execution_nodes(
notes = list(state.get("notes", []))
pending_permission = state.get("pending_permission")
warnings = list(state.get("warnings", []))
shown = list(state.get("drafts_shown", []))
drafts = dict(state.get("drafts", {}))
outcomes = state.get("outcomes", [])
@@ -217,6 +219,9 @@ def make_execution_nodes(
"content": record["draft"],
"attachments": record.get("draft_attachments", {}),
}
# The user is being shown this Deliverable.
if kind.value not in shown:
shown.append(kind.value)
questions = record.get("questions") or [
(
f"The {record['role']} stopped: {record['summary']} "
@@ -229,6 +234,7 @@ def make_execution_nodes(
update: dict[str, Any] = {
"deliverables": deliverables,
"drafts": drafts,
"drafts_shown": shown,
"pending_questions": pending_questions,
"pending_permission": pending_permission,
"outcomes_applied": len(outcomes),
@@ -434,6 +440,11 @@ def make_execution_nodes(
working=working,
library=task_input.get("library"),
review=review if submission_review else None,
# With a user present, their first sight of a Deliverable comes
# before it is submitted, whatever the Specialist intends.
confirm_first=confirm_drafts
and not task_input["context"].get("pre_approved", True)
and not task_input.get("shown"),
),
refusals,
role.name,
@@ -559,7 +570,7 @@ def make_execution_nodes(
async def finalize(state: ExecutionState) -> dict[str, Any]:
ledger = Ledger.from_dict(state["ledger"])
_write_outputs(output_root, state)
_write_outputs(output_root, state, "completed", ledger)
close(state, "completed", has_files=True)
await record_progress(state, ledger, "completed")
await record_lessons(state, ledger, "completed")
@@ -567,7 +578,10 @@ def make_execution_nodes(
async def end_with_warnings(state: ExecutionState) -> dict[str, Any]:
ledger = Ledger.from_dict(state["ledger"])
close(state, "completed_with_warnings", has_files=False)
# What was produced is kept, marked as unfinished: a person deciding what to
# do next needs to read it, and it is the evidence of what went wrong.
wrote = _write_outputs(output_root, state, "completed_with_warnings", ledger)
close(state, "completed_with_warnings", has_files=wrote)
await record_progress(state, ledger, "completed_with_warnings")
await record_lessons(state, ledger, "completed_with_warnings")
return {"status": "completed_with_warnings"}
@@ -706,6 +720,9 @@ def _specialist_input(
"attempt": attempt,
"allow_cloud_read": state.get("allow_cloud_read"),
"working": dict(state.get("working", {}).get(kind.value, {})),
# Whether the user has been shown a draft of this Deliverable. Answering some
# other question about it, such as what to do after it stopped, is not that.
"shown": kind.value in state.get("drafts_shown", []),
"context": {
"workflow_request": state["workflow_request"],
"pre_approved": bool(state.get("pre_approved")),
@@ -762,7 +779,14 @@ def automatic_reply(
prompt: Mapping[str, Any], scripted_answers: Sequence[Sequence[str]] = ()
) -> dict[str, Any]:
"""The reply in a pre-approved Execution: approve, deny cloud reads, and answer
questions from scripted (pattern, answer) pairs where one matches."""
questions from scripted answers where one matches.
A scripted answer is (pattern, answer) or (pattern, answer, asker). It answers a
question its pattern matches; with an asker, only when that pattern also matches
who is asking (a Deliverable kind, or the orchestrator). The first that applies
wins. So facts written for requirements discovery are not given to a later
Specialist that only wants its draft confirmed.
"""
if "approval" in prompt:
return {
"approval": {"phase": prompt["approval"]["phase"], "decision": "approve"}
@@ -772,11 +796,13 @@ def automatic_reply(
lines = []
unanswered = False
for item in prompt.get("questions", []):
asker = str(item.get("from", ""))
answer = next(
(
scripted
for pattern, scripted in scripted_answers
if re.search(pattern, item["question"], re.IGNORECASE)
scripted[1]
for scripted in scripted_answers
if re.search(scripted[0], item["question"], re.IGNORECASE)
and (len(scripted) < 3 or re.search(scripted[2], asker, re.IGNORECASE))
),
None,
)
@@ -839,13 +865,21 @@ def _case(state: ExecutionState) -> str:
return state.get("customer_case") or UNASSIGNED_CASE
def _write_outputs(output_root: Path, state: ExecutionState) -> None:
"""Write a finished Execution's files under its Customer case, with a manifest
that ties the request to what was generated for it."""
def _write_outputs(
output_root: Path, state: ExecutionState, status: str, ledger: Ledger
) -> bool:
"""Write an Execution's files under its Customer case, with a manifest that ties
the request to what was generated for it. Returns whether anything was written.
An Execution that ended with warnings is written too. Its manifest carries the
warnings and each Deliverable's status, so nothing unapproved reads as final.
"""
customer_case = _case(state)
execution_id = state["execution_id"]
out_dir = execution_dir(output_root, execution_id, customer_case)
deliverables: Mapping[str, Mapping[str, Any]] = state.get("deliverables", {})
if not deliverables:
return False
files: dict[str, str] = {}
by_kind: dict[str, dict[str, Any]] = {}
guide = deliverables.get(DeliverableKind.SOLUTION_GUIDE.value)
@@ -856,7 +890,12 @@ def _write_outputs(output_root: Path, state: ExecutionState) -> None:
files[document] = deliverable["content"]
attachments = dict(deliverable.get("attachments", {}))
files.update(attachments)
by_kind[kind] = {"document": document, "attachments": sorted(attachments)}
by_kind[kind] = {
"document": document,
"attachments": sorted(attachments),
# submitted, or back in draft after something it depends on was reopened
"status": ledger.status(DeliverableKind(kind)).value,
}
for name, content in files.items():
path = out_dir / name
path.parent.mkdir(parents=True, exist_ok=True)
@@ -868,7 +907,11 @@ def _write_outputs(output_root: Path, state: ExecutionState) -> None:
"session_id": state.get("session_id"),
"intent_version": state.get("intent_version"),
"workflow_request": state["workflow_request"],
"status": "completed",
"status": status,
"warnings": list(state.get("warnings", [])),
"phases": {
phase.value: ledger.phase_status(phase).value for phase in PHASE_ORDER
},
"solution_guide": GUIDE_FILENAME if guide else None,
"deliverables": by_kind,
}
@@ -876,3 +919,4 @@ def _write_outputs(output_root: Path, state: ExecutionState) -> None:
(out_dir / MANIFEST_FILENAME).write_text(
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
)
return True

122
app/nodes/intake.py Normal file
View File

@@ -0,0 +1,122 @@
"""Intake: what a message that starts something is asking for (ADR-0007).
Inside an Execution the router reads the user's replies. This is the same reading
one step earlier, for a message that arrives with no Execution in progress:
new request design this workload start an Execution
change request change what was delivered (ADR-0006; not built yet)
question what can you do, how long... answer; start nothing
Without a decision, and whenever the reading is not confident, the message is taken
as a new request: that is what every message was before, and starting work nobody
asked for is cheaper to put right than refusing work somebody did.
"""
from __future__ import annotations
import logging
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from app.adk.runtime import Decider, Question
from app.customer_case import IN_PROGRESS, read_history
from app.states.execution_state import UNASSIGNED_CASE
logger = logging.getLogger(__name__)
NEW_REQUEST = "new_request"
CHANGE_REQUEST = "change_request"
QUESTION = "question"
ABOUT = (
"I design cloud solution architectures. Tell me, at a high level, what you want "
"built, and I will work through requirements, a target architecture, "
"infrastructure code, validation and a packaged guide, asking for your approval "
"at each Phase. Nothing was started for this message."
)
CHANGE_NOTE = (
"This reads as a change to work already delivered for this Customer case. "
"Changing an earlier architecture in place is not available yet, so this is "
"being designed as a new request."
)
_MESSAGE_LIMIT = 8_000
_EARLIER_SHOWN = 5
@dataclass(frozen=True)
class IntakeReading:
kind: str # new_request | change_request | question
confidence: float = 0.0
judged: bool = False # False when no decision was available
earlier: int = 0 # finished Executions of the same Customer case
def finished_intents(output_root: Path, customer_case: str) -> list[Mapping[str, Any]]:
"""The case's earlier statements of intent whose Executions delivered something."""
if not customer_case or customer_case == UNASSIGNED_CASE:
return []
return [
item
for item in read_history(output_root, customer_case)["intents"]
if isinstance(item, Mapping)
and str(item.get("status", IN_PROGRESS)).startswith("completed")
]
class Intake:
def __init__(
self,
decider: Decider,
output_root: Path,
*,
act: float = 0.75,
) -> None:
self._decider = decider
self._output_root = output_root
# How sure the reading must be before anything but a new Execution is done.
self._act = act
async def read(
self, message: str, customer_case: str | None = None, scope: str = ""
) -> IntakeReading:
earlier = finished_intents(self._output_root, customer_case or "")
text = message.strip()
if not text:
return IntakeReading(NEW_REQUEST, earlier=len(earlier))
answers = await self._decider.decide(
_state(text, earlier),
{
"intake": Question(
"choice",
"What is the user's message? new_request: they describe "
"something they want designed or built. change_request: they "
"ask to change or extend an architecture already delivered to "
"them. question: they ask about the service or about earlier "
"work, and want an answer, not new work.",
(NEW_REQUEST, CHANGE_REQUEST, QUESTION),
)
},
scope=scope,
)
answer = (answers or {}).get("intake")
if answer is None:
return IntakeReading(NEW_REQUEST, earlier=len(earlier))
kind, confidence = str(answer.value), float(answer.confidence)
if kind != NEW_REQUEST and confidence < self._act:
kind = NEW_REQUEST # not sure enough to do anything but start
if kind == CHANGE_REQUEST and not earlier:
kind = NEW_REQUEST # there is nothing delivered to change
logger.info("[%s] intake: %s (%.2f)", scope, kind, confidence)
return IntakeReading(kind, confidence, judged=True, earlier=len(earlier))
def _state(message: str, earlier: Sequence[Mapping[str, Any]]) -> str:
lines = [f"Architectures already delivered to this user: {len(earlier)}."]
for item in list(earlier)[-_EARLIER_SHOWN:]:
request = " ".join(str(item.get("workflow_request", "")).split())[:300]
lines.append(f"- {request}")
lines.append(f"The user's message: {message[:_MESSAGE_LIMIT]}")
return "\n".join(lines)

View File

@@ -139,6 +139,9 @@ class Readings:
judged: dict[str, Any] = field(default_factory=dict) # what the decider answered
# How much of the agreed requirements is shown to the decider with a user's message.
_AGREED_LIMIT = 24_000
# The actions that reopen a Deliverable, and which one.
_REOPENS = (
"revise_requested",
@@ -356,6 +359,15 @@ class Router:
# our own question about a change they already asked for.
structured = reply.get("kind") == "changes" or bool(pending)
agreed = (
""
if structured
else str(
view.deliverables.get(DeliverableKind.REQUIREMENTS.value, {})
.get("attachments", {})
.get(REQUIREMENTS_FILE, "")
)[:_AGREED_LIMIT]
)
questions = {
"change_target": Question(
"choice",
@@ -375,9 +387,18 @@ class Router:
"Is the user asking for a change to work that has already been "
"submitted, as opposed to answering a question or adding information?",
)
if agreed:
# Someone who restates what is already agreed is answering, however
# much it sounds like a requirement.
questions["already_agreed"] = Question(
"noul",
"Does the user's message only confirm or repeat what the agreed "
"requirements above already say, asking for nothing different?",
)
state = (
f"Submitted Deliverables: {', '.join(kind.value for kind in candidates)}.\n"
f"Questions waiting for the user: {len(view.pending_questions)}.\n"
+ (f"The agreed requirements:\n{agreed}\n" if agreed else "")
+ f"Questions waiting for the user: {len(view.pending_questions)}.\n"
# What we asked is background. Only the user's own words are their
# message: our question echoed back is not something they said.
+ (
@@ -412,6 +433,10 @@ class Router:
}
probability = 1.0 if structured else float(answers["asks_change"].value)
disposition = self._thresholds.disposition(probability)
restated = answers.get("already_agreed")
if restated is not None and restated.is_yes(self._thresholds.act):
# It restates what is agreed: an answer, not a change.
disposition = CLOSE
target_answer = answers["change_target"]
target = DeliverableKind(str(target_answer.value))
readings.change_target = target

View File

@@ -67,6 +67,7 @@ class ExecutionState(TypedDict, total=False):
pending_questions: list[dict[str, Any]] # {kind, questions} not yet asked
awaiting_questions: list[dict[str, Any]] # {kind, questions} asked, not answered
answers: dict[str, list[dict[str, Any]]] # kind -> [{questions, reply}]
drafts_shown: list[str] # Deliverables a draft of which the user has been shown
drafts: dict[
str, dict[str, Any]
] # kind -> {content, attachments} awaiting the user

View File

@@ -21,6 +21,7 @@ from app.adk.runtime import (
files_from_argument,
)
from app.states.ledger import Ledger, LedgerRefusal, PhaseStatus
from app.tools.pillars import REVIEW_HEADING, render_checklist_review
from app.tools.workspace import (
FILES,
basis_of,
@@ -269,6 +270,7 @@ def specialist_tools(
working: MutableMapping[str, Any] | None = None,
library: Mapping[str, Mapping[str, Any]] | None = None,
review: Callable[[Mapping[str, str]], Awaitable[Sequence[str]]] | None = None,
confirm_first: bool = False,
) -> list[ToolSpec]:
"""A Specialist's tools: submit, ask, its working copy, and its role's tools with
the cloud-read gate.
@@ -279,6 +281,9 @@ def specialist_tools(
given to the Specialist as data files alone, to read whole when it needs one.
`review` looks at the meaning of a structurally complete submission; what it
raises is returned to the Specialist once per run, and never a second time.
`confirm_first` is set while a user is present and has not yet been asked about
this Deliverable: a submission is then refused until the Specialist has put its
draft to them. A rule, because an instruction alone was not always followed.
"""
working = {} if working is None else working
is_graph = kind is DeliverableKind.TARGET_ARCHITECTURE
@@ -339,6 +344,21 @@ def specialist_tools(
"it is already covered. This review runs once, so your next "
"submission is accepted."
)
if confirm_first:
return ToolResult(
"Not submitted. The user has not seen this Deliverable yet. Call "
"`needs_input` with a short summary asking them to approve it or "
"request changes, passing this content as `draft` with its "
"attachments. What you have recorded is kept. Submit when they "
"have replied."
)
if kind is DeliverableKind.DESIGN_RECOMMENDATIONS:
# The pillar advisors' checklist reviews, written from what they
# recorded. A Revision replaces the section, never stacks a second.
checklist_review = render_checklist_review(working)
content = content.split(f"\n## {REVIEW_HEADING}")[0].rstrip() + "\n"
if checklist_review:
content += "\n" + checklist_review
if is_graph:
# The tables and the diagram are views of the graph, so the engine
# writes them: they cannot disagree with it.

View File

@@ -3,6 +3,12 @@
Pillar advice is working material for the design recommendations Deliverable, not a
Deliverable itself, so it never touches the Ledger. It is kept in that Deliverable's
working memory, so a Revision re-consults only the pillars its feedback touches.
Each advisor also returns its review of the pillar's own checklist: for every item,
whether its advice addresses it, or why it does not apply. The reviews are kept
beside the advice and written into the Deliverable by code when it is submitted, so
an architect sees the whole checklist accounted for and no model spends output
copying it.
"""
from __future__ import annotations
@@ -26,6 +32,14 @@ from app.adk.runtime import (
CONSULT_TOOL = "consult_pillar_advisors"
NO_GUIDANCE = "No guidance"
ADVICE_KEY = "pillar_advice" # working memory: pillar -> the advice last given
CHECKLIST_KEY = "pillar_checklists" # working memory: pillar -> its checklist review
REVIEW_HEADING = "Architecture framework checklist review"
STATUSES = ("addressed", "not_applicable", "not_addressed")
_STATUS_WORDS = {
"addressed": "Addressed",
"not_applicable": "Does not apply",
"not_addressed": "Not addressed",
}
def pillar_consultation_tool(
@@ -42,7 +56,7 @@ def pillar_consultation_tool(
returns the remembered advice for the rest.
"""
async def advise(pillar: str, brief: str) -> str:
async def advise(pillar: str, brief: str) -> tuple[str, list[dict[str, str]]]:
async with AsyncExitStack() as stack:
if limiter:
await stack.enter_async_context(limiter.slot())
@@ -71,12 +85,16 @@ def pillar_consultation_tool(
remembered: dict[str, str] = (
dict(memory.get(ADVICE_KEY, {})) if memory is not None else {}
)
reviews: dict[str, list[dict[str, str]]] = (
dict(memory.get(CHECKLIST_KEY, {})) if memory is not None else {}
)
sections = []
for pillar in pillar_skills:
if pillar in fresh:
advice = fresh[pillar]
advice, review = fresh[pillar]
if not advice.startswith(NO_GUIDANCE):
remembered[pillar] = advice
reviews[pillar] = review
sections.append(f"## {pillar}\n\n{advice}")
elif pillar in remembered:
sections.append(
@@ -85,6 +103,7 @@ def pillar_consultation_tool(
)
if memory is not None:
memory[ADVICE_KEY] = remembered
memory[CHECKLIST_KEY] = reviews
return ToolResult("\n\n".join(sections))
return ToolSpec(
@@ -115,36 +134,116 @@ async def _advise(
skill: str,
brief: str,
extra_tools: Sequence[ToolSpec],
) -> str:
) -> tuple[str, list[dict[str, str]]]:
"""One advisor's advice, and its review of the pillar's checklist."""
role = dataclasses.replace(advisor, skills=(skill,))
task = AgentTask(
role=advisor.name,
prompt=f"Advise on the {pillar} pillar for this workload.",
context={"pillar": pillar, "brief": brief},
)
review: list[dict[str, str]] = []
try:
outcome = await runtime.run(role, task, [_submit_advice_tool(), *extra_tools])
outcome = await runtime.run(
role, task, [_submit_advice_tool(review), *extra_tools]
)
except Exception as exc: # noqa: BLE001 - one failed advisor must not lose the others
return f"{NO_GUIDANCE}: the advisor failed ({exc})."
return f"{NO_GUIDANCE}: the advisor failed ({exc}).", []
if isinstance(outcome, Submitted):
return outcome.content.strip()
return f"{NO_GUIDANCE}: the advisor ended with {type(outcome).__name__}."
return outcome.content.strip(), review
return f"{NO_GUIDANCE}: the advisor ended with {type(outcome).__name__}.", []
def _submit_advice_tool() -> ToolSpec:
async def submit_advice(content: str) -> ToolResult:
def _submit_advice_tool(review: list[dict[str, str]]) -> ToolSpec:
"""`review` receives the checklist review the advisor submits with its advice."""
async def submit_advice(content: str, checklist: Sequence[Any] = ()) -> ToolResult:
if not content.strip():
return ToolResult("Not submitted: the advice is empty.")
review[:] = [
{
"item": " ".join(str(entry.get("item", "")).split()),
"status": str(entry.get("status", "")),
"note": " ".join(str(entry.get("note", "")).split()),
}
for entry in checklist or ()
if isinstance(entry, Mapping)
and str(entry.get("item", "")).strip()
and entry.get("status") in STATUSES
]
return ToolResult("Advice submitted.", Submitted(content))
text = {"type": "string"}
return ToolSpec(
"submit_advice",
"Submit your pillar recommendations as markdown. Ends your turn.",
"Submit your pillar recommendations as markdown, with your review of the "
"pillar's checklist. Ends your turn.",
{
"type": "object",
"properties": {"content": {"type": "string"}},
"properties": {
"content": text,
"checklist": {
"type": "array",
"description": "One entry for every item of the Skill's "
"validation checklist.",
"items": {
"type": "object",
"properties": {
"item": text,
"status": {"type": "string", "enum": list(STATUSES)},
"note": {
"type": "string",
"description": "How your advice addresses it, or "
"why it does not apply to this workload.",
},
},
"required": ["item", "status", "note"],
},
},
},
"required": ["content"],
},
submit_advice,
tier="execution",
)
def render_checklist_review(working: Mapping[str, Any]) -> str:
"""The advisors' checklist reviews as one section of markdown tables, or "" when
there are none. Written by code from what the advisors recorded."""
reviews = working.get(CHECKLIST_KEY)
if not isinstance(reviews, Mapping) or not any(reviews.values()):
return ""
parts = [
f"## {REVIEW_HEADING}",
"",
(
"Each pillar advisor's review of its pillar's checklist for this "
"workload: what its advice addresses, and what does not apply and why."
),
]
for pillar, entries in reviews.items():
if not entries:
continue
parts += [
"",
f"### {pillar}",
"",
"| Checklist item | Review | How, or why not |",
"| --- | --- | --- |",
]
parts += [
"| "
+ " | ".join(
str(value).replace("|", "\\|")
for value in (
entry.get("item", ""),
_STATUS_WORDS.get(str(entry.get("status")), ""),
entry.get("note", ""),
)
)
+ " |"
for entry in entries
if isinstance(entry, Mapping)
]
return "\n".join(parts) + "\n"

View File

@@ -0,0 +1,78 @@
"""A review of the requirements: they describe the problem, not the solution.
Requirements discovery is kept apart from solutioning, so a requirement names no
product. That rule was an instruction the engine could not enforce; the evaluation
checked it only afterwards, against a list of product names. This asks the decider
(ADR-0007), once per requirement, whether it names a cloud product or managed
service. It never blocks: what it raises is returned to the Specialist once.
"""
from __future__ import annotations
import logging
from collections.abc import Mapping, Sequence
from typing import Any
from app.adk.runtime import Decider, Question
from app.workflows.chain import REQUIREMENTS_FILE, read_list
from app.workflows.deliverables import DeliverableKind
logger = logging.getLogger(__name__)
_TEXT = 200
class RequirementsReview:
def __init__(self, decider: Decider, *, act: float = 0.3) -> None:
self._decider = decider
# Lower than the router's threshold, from measurement on 2026-10-08: across
# 61 requirements from three passed runs the highest answer was 0.10, and
# requirements written to name a product scored 0.34, 0.61 and 0.93.
self._act = act
async def __call__(
self,
kind: DeliverableKind,
attachments: Mapping[str, str],
upstream: Mapping[str, Mapping[str, Any]] | None,
scope: str = "",
) -> Sequence[str]:
"""The requirements that name a product, each as a line for the Specialist."""
if kind is not DeliverableKind.REQUIREMENTS:
return ()
items = [
item
for item in read_list(
{"requirements": {"attachments": attachments}},
"requirements",
REQUIREMENTS_FILE,
)
if isinstance(item.get("id"), str) and isinstance(item.get("text"), str)
]
if not items:
return ()
answers = await self._decider.decide(
"Requirements for a workload, written before any product is chosen.",
{
item["id"]: Question(
"noul",
f'Requirement: "{item["text"]}". Does it name a specific cloud '
"provider's product or managed service, as opposed to "
"describing what is needed?",
)
for item in items
},
scope=scope,
)
if not answers:
return ()
raised = [
f'{item["id"]} names a product or service: "{item["text"][:_TEXT]}". '
"Say what is needed, not what to use"
for item in items
if (answer := answers.get(item["id"])) is not None
and answer.is_yes(self._act)
]
if raised:
logger.info("[%s] requirements review raised %d", scope, len(raised))
return tuple(raised)

View File

@@ -153,19 +153,36 @@ def conditions_of(item: Mapping[str, Any]) -> tuple[str, ...]:
return (text.strip(),) if isinstance(text, str) and text.strip() else ()
OUTSIDE_CODE = "outside_code"
def outside_code_of(item: Mapping[str, Any]) -> tuple[str, ...]:
"""What a requirement asks for that infrastructure code cannot show: an outcome
measured in operation, application behaviour, or a process people follow."""
listed = item.get(OUTSIDE_CODE)
if not isinstance(listed, list):
return ()
return tuple(c.strip() for c in listed if isinstance(c, str) and c.strip())
def _lacking_conditions(items: Sequence[Mapping[str, Any]], name: str) -> list[str]:
"""A constraint must say, one by one, what would have to be true for it to be met."""
"""A constraint must say, one by one, what would have to be true for it to be met:
under `conditions` what infrastructure code can show, under `outside_code` what
it cannot. At least one of the two has an entry."""
problems = []
for item in items:
listed = item.get("conditions")
if (
not isinstance(listed, list)
or not listed
or not all(isinstance(c, str) and c.strip() for c in listed)
):
outside = item.get(OUTSIDE_CODE, [])
well_formed = all(
isinstance(group, list)
and all(isinstance(c, str) and c.strip() for c in group)
for group in (listed, outside)
)
if not well_formed or not (listed or outside):
problems.append(
f"{name}: {item['id']} needs `conditions`: a list with one entry "
"for each separate thing it asks for"
"for each separate thing it asks for that infrastructure code can "
f"show (and `{OUTSIDE_CODE}` for what it cannot)"
)
return problems

View File

@@ -43,6 +43,7 @@ def build_execution_graph(
skill_versions: Mapping[str, str] | None = None,
orchestrator: OrchestratorPolicy | None = None,
submission_review: SubmissionReview | None = None,
confirm_drafts: bool = False,
) -> StateGraph:
nodes = make_execution_nodes(
runtime=runtime,
@@ -55,6 +56,7 @@ def build_execution_graph(
skill_versions=skill_versions,
orchestrator=orchestrator,
submission_review=submission_review,
confirm_drafts=confirm_drafts,
)
graph = StateGraph(ExecutionState)

View File

@@ -195,7 +195,8 @@ def chain_gaps(
A functional requirement is carried by a component that has an Element, or by a
Check of its own. A non-functional requirement and a design recommendation are
each a constraint, so each must be the source of a Check (a recommendation may
instead be marked not applicable, with a reason).
instead be marked not applicable, with a reason). A constraint that lists
nothing infrastructure code can show, only what lies outside it, needs none.
"""
gaps: list[str] = []
requirement_ids = {item["id"] for item in requirements}
@@ -229,6 +230,8 @@ def chain_gaps(
)
if requirement.get("kind") == "functional" and served:
continue
if _only_outside_code(requirement):
continue
gaps.append(f"Requirement {requirement['id']} has no Check")
for selection in products:
used = any(
@@ -249,11 +252,19 @@ def chain_gaps(
"which is not a design recommendation"
)
for recommendation in recommendations:
if recommendation["id"] not in sources | excused:
if recommendation["id"] not in sources | excused and not _only_outside_code(
recommendation
):
gaps.append(f"Recommendation {recommendation['id']} has no Check")
return tuple(gaps)
def _only_outside_code(item: Mapping[str, Any]) -> bool:
"""Whether a constraint says, explicitly, that nothing it asks for can be shown
by infrastructure code."""
return item.get("conditions") == [] and bool(item.get("outside_code"))
# --- the diagram is a view of the graph ----------------------------------------------
_UNSAFE = re.compile(r'["\[\]{}()<>|`#;]')