feat: all four evaluation cases pass; intake, checklist review, thresholds
Case 04 (interactive) - A scripted answer may name who it is for, so facts written for requirements discovery are not given to a Specialist confirming a draft. - A reply that restates what the agreed requirements already say is read as an answer, not a request for changes. - With a user present, a first submission is refused until the user has been shown a draft of that Deliverable (CONFIRM_DRAFTS). Routing and reviews - Intake: a message that starts something is read first. A question is answered and starts nothing; a change to delivered work starts as a new request with a note, until Change requests are built. - Requirements and recommendations list what infrastructure code can show (conditions) apart from what it cannot (outside_code); the latter needs no Check and validation proposes nothing for it. - A requirements review, "names no products", behind its own switch (off). - Each pillar advisor reviews its pillar's checklist; the engine writes the reviews into the design recommendations. The judged checklist check went from 9-45% a pillar to 100% in one run of case 04. Evaluation - Per-case thresholds for Revisions, seconds and tokens, set from every completed run on this build. - A progress file after every agent run, so an interrupted run keeps its time and tokens. - eval/calibrate_judge.py with a 32-question set awaiting a person's labels. Unlabelled: Jev stable at 1.00, Gemini 0.92, agreeing on 0.91. Smaller - An Execution that ends with warnings writes its Deliverables, with the warnings and each status in the manifest. - CI installs the application's dependencies before linting and testing. - A load-balancer pattern in the IaC patterns Skill. Plans record nine runs of 2026-10-08. Iteration 6 is complete. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -21,6 +21,7 @@ from app.adk.runtime import (
|
||||
files_from_argument,
|
||||
)
|
||||
from app.states.ledger import Ledger, LedgerRefusal, PhaseStatus
|
||||
from app.tools.pillars import REVIEW_HEADING, render_checklist_review
|
||||
from app.tools.workspace import (
|
||||
FILES,
|
||||
basis_of,
|
||||
@@ -269,6 +270,7 @@ def specialist_tools(
|
||||
working: MutableMapping[str, Any] | None = None,
|
||||
library: Mapping[str, Mapping[str, Any]] | None = None,
|
||||
review: Callable[[Mapping[str, str]], Awaitable[Sequence[str]]] | None = None,
|
||||
confirm_first: bool = False,
|
||||
) -> list[ToolSpec]:
|
||||
"""A Specialist's tools: submit, ask, its working copy, and its role's tools with
|
||||
the cloud-read gate.
|
||||
@@ -279,6 +281,9 @@ def specialist_tools(
|
||||
given to the Specialist as data files alone, to read whole when it needs one.
|
||||
`review` looks at the meaning of a structurally complete submission; what it
|
||||
raises is returned to the Specialist once per run, and never a second time.
|
||||
`confirm_first` is set while a user is present and has not yet been asked about
|
||||
this Deliverable: a submission is then refused until the Specialist has put its
|
||||
draft to them. A rule, because an instruction alone was not always followed.
|
||||
"""
|
||||
working = {} if working is None else working
|
||||
is_graph = kind is DeliverableKind.TARGET_ARCHITECTURE
|
||||
@@ -339,6 +344,21 @@ def specialist_tools(
|
||||
"it is already covered. This review runs once, so your next "
|
||||
"submission is accepted."
|
||||
)
|
||||
if confirm_first:
|
||||
return ToolResult(
|
||||
"Not submitted. The user has not seen this Deliverable yet. Call "
|
||||
"`needs_input` with a short summary asking them to approve it or "
|
||||
"request changes, passing this content as `draft` with its "
|
||||
"attachments. What you have recorded is kept. Submit when they "
|
||||
"have replied."
|
||||
)
|
||||
if kind is DeliverableKind.DESIGN_RECOMMENDATIONS:
|
||||
# The pillar advisors' checklist reviews, written from what they
|
||||
# recorded. A Revision replaces the section, never stacks a second.
|
||||
checklist_review = render_checklist_review(working)
|
||||
content = content.split(f"\n## {REVIEW_HEADING}")[0].rstrip() + "\n"
|
||||
if checklist_review:
|
||||
content += "\n" + checklist_review
|
||||
if is_graph:
|
||||
# The tables and the diagram are views of the graph, so the engine
|
||||
# writes them: they cannot disagree with it.
|
||||
|
||||
@@ -3,6 +3,12 @@
|
||||
Pillar advice is working material for the design recommendations Deliverable, not a
|
||||
Deliverable itself, so it never touches the Ledger. It is kept in that Deliverable's
|
||||
working memory, so a Revision re-consults only the pillars its feedback touches.
|
||||
|
||||
Each advisor also returns its review of the pillar's own checklist: for every item,
|
||||
whether its advice addresses it, or why it does not apply. The reviews are kept
|
||||
beside the advice and written into the Deliverable by code when it is submitted, so
|
||||
an architect sees the whole checklist accounted for and no model spends output
|
||||
copying it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -26,6 +32,14 @@ from app.adk.runtime import (
|
||||
CONSULT_TOOL = "consult_pillar_advisors"
|
||||
NO_GUIDANCE = "No guidance"
|
||||
ADVICE_KEY = "pillar_advice" # working memory: pillar -> the advice last given
|
||||
CHECKLIST_KEY = "pillar_checklists" # working memory: pillar -> its checklist review
|
||||
REVIEW_HEADING = "Architecture framework checklist review"
|
||||
STATUSES = ("addressed", "not_applicable", "not_addressed")
|
||||
_STATUS_WORDS = {
|
||||
"addressed": "Addressed",
|
||||
"not_applicable": "Does not apply",
|
||||
"not_addressed": "Not addressed",
|
||||
}
|
||||
|
||||
|
||||
def pillar_consultation_tool(
|
||||
@@ -42,7 +56,7 @@ def pillar_consultation_tool(
|
||||
returns the remembered advice for the rest.
|
||||
"""
|
||||
|
||||
async def advise(pillar: str, brief: str) -> str:
|
||||
async def advise(pillar: str, brief: str) -> tuple[str, list[dict[str, str]]]:
|
||||
async with AsyncExitStack() as stack:
|
||||
if limiter:
|
||||
await stack.enter_async_context(limiter.slot())
|
||||
@@ -71,12 +85,16 @@ def pillar_consultation_tool(
|
||||
remembered: dict[str, str] = (
|
||||
dict(memory.get(ADVICE_KEY, {})) if memory is not None else {}
|
||||
)
|
||||
reviews: dict[str, list[dict[str, str]]] = (
|
||||
dict(memory.get(CHECKLIST_KEY, {})) if memory is not None else {}
|
||||
)
|
||||
sections = []
|
||||
for pillar in pillar_skills:
|
||||
if pillar in fresh:
|
||||
advice = fresh[pillar]
|
||||
advice, review = fresh[pillar]
|
||||
if not advice.startswith(NO_GUIDANCE):
|
||||
remembered[pillar] = advice
|
||||
reviews[pillar] = review
|
||||
sections.append(f"## {pillar}\n\n{advice}")
|
||||
elif pillar in remembered:
|
||||
sections.append(
|
||||
@@ -85,6 +103,7 @@ def pillar_consultation_tool(
|
||||
)
|
||||
if memory is not None:
|
||||
memory[ADVICE_KEY] = remembered
|
||||
memory[CHECKLIST_KEY] = reviews
|
||||
return ToolResult("\n\n".join(sections))
|
||||
|
||||
return ToolSpec(
|
||||
@@ -115,36 +134,116 @@ async def _advise(
|
||||
skill: str,
|
||||
brief: str,
|
||||
extra_tools: Sequence[ToolSpec],
|
||||
) -> str:
|
||||
) -> tuple[str, list[dict[str, str]]]:
|
||||
"""One advisor's advice, and its review of the pillar's checklist."""
|
||||
role = dataclasses.replace(advisor, skills=(skill,))
|
||||
task = AgentTask(
|
||||
role=advisor.name,
|
||||
prompt=f"Advise on the {pillar} pillar for this workload.",
|
||||
context={"pillar": pillar, "brief": brief},
|
||||
)
|
||||
review: list[dict[str, str]] = []
|
||||
try:
|
||||
outcome = await runtime.run(role, task, [_submit_advice_tool(), *extra_tools])
|
||||
outcome = await runtime.run(
|
||||
role, task, [_submit_advice_tool(review), *extra_tools]
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - one failed advisor must not lose the others
|
||||
return f"{NO_GUIDANCE}: the advisor failed ({exc})."
|
||||
return f"{NO_GUIDANCE}: the advisor failed ({exc}).", []
|
||||
if isinstance(outcome, Submitted):
|
||||
return outcome.content.strip()
|
||||
return f"{NO_GUIDANCE}: the advisor ended with {type(outcome).__name__}."
|
||||
return outcome.content.strip(), review
|
||||
return f"{NO_GUIDANCE}: the advisor ended with {type(outcome).__name__}.", []
|
||||
|
||||
|
||||
def _submit_advice_tool() -> ToolSpec:
|
||||
async def submit_advice(content: str) -> ToolResult:
|
||||
def _submit_advice_tool(review: list[dict[str, str]]) -> ToolSpec:
|
||||
"""`review` receives the checklist review the advisor submits with its advice."""
|
||||
|
||||
async def submit_advice(content: str, checklist: Sequence[Any] = ()) -> ToolResult:
|
||||
if not content.strip():
|
||||
return ToolResult("Not submitted: the advice is empty.")
|
||||
review[:] = [
|
||||
{
|
||||
"item": " ".join(str(entry.get("item", "")).split()),
|
||||
"status": str(entry.get("status", "")),
|
||||
"note": " ".join(str(entry.get("note", "")).split()),
|
||||
}
|
||||
for entry in checklist or ()
|
||||
if isinstance(entry, Mapping)
|
||||
and str(entry.get("item", "")).strip()
|
||||
and entry.get("status") in STATUSES
|
||||
]
|
||||
return ToolResult("Advice submitted.", Submitted(content))
|
||||
|
||||
text = {"type": "string"}
|
||||
return ToolSpec(
|
||||
"submit_advice",
|
||||
"Submit your pillar recommendations as markdown. Ends your turn.",
|
||||
"Submit your pillar recommendations as markdown, with your review of the "
|
||||
"pillar's checklist. Ends your turn.",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"content": {"type": "string"}},
|
||||
"properties": {
|
||||
"content": text,
|
||||
"checklist": {
|
||||
"type": "array",
|
||||
"description": "One entry for every item of the Skill's "
|
||||
"validation checklist.",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"item": text,
|
||||
"status": {"type": "string", "enum": list(STATUSES)},
|
||||
"note": {
|
||||
"type": "string",
|
||||
"description": "How your advice addresses it, or "
|
||||
"why it does not apply to this workload.",
|
||||
},
|
||||
},
|
||||
"required": ["item", "status", "note"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["content"],
|
||||
},
|
||||
submit_advice,
|
||||
tier="execution",
|
||||
)
|
||||
|
||||
|
||||
def render_checklist_review(working: Mapping[str, Any]) -> str:
|
||||
"""The advisors' checklist reviews as one section of markdown tables, or "" when
|
||||
there are none. Written by code from what the advisors recorded."""
|
||||
reviews = working.get(CHECKLIST_KEY)
|
||||
if not isinstance(reviews, Mapping) or not any(reviews.values()):
|
||||
return ""
|
||||
parts = [
|
||||
f"## {REVIEW_HEADING}",
|
||||
"",
|
||||
(
|
||||
"Each pillar advisor's review of its pillar's checklist for this "
|
||||
"workload: what its advice addresses, and what does not apply and why."
|
||||
),
|
||||
]
|
||||
for pillar, entries in reviews.items():
|
||||
if not entries:
|
||||
continue
|
||||
parts += [
|
||||
"",
|
||||
f"### {pillar}",
|
||||
"",
|
||||
"| Checklist item | Review | How, or why not |",
|
||||
"| --- | --- | --- |",
|
||||
]
|
||||
parts += [
|
||||
"| "
|
||||
+ " | ".join(
|
||||
str(value).replace("|", "\\|")
|
||||
for value in (
|
||||
entry.get("item", ""),
|
||||
_STATUS_WORDS.get(str(entry.get("status")), ""),
|
||||
entry.get("note", ""),
|
||||
)
|
||||
)
|
||||
+ " |"
|
||||
for entry in entries
|
||||
if isinstance(entry, Mapping)
|
||||
]
|
||||
return "\n".join(parts) + "\n"
|
||||
|
||||
78
app/tools/requirements_review.py
Normal file
78
app/tools/requirements_review.py
Normal file
@@ -0,0 +1,78 @@
|
||||
"""A review of the requirements: they describe the problem, not the solution.
|
||||
|
||||
Requirements discovery is kept apart from solutioning, so a requirement names no
|
||||
product. That rule was an instruction the engine could not enforce; the evaluation
|
||||
checked it only afterwards, against a list of product names. This asks the decider
|
||||
(ADR-0007), once per requirement, whether it names a cloud product or managed
|
||||
service. It never blocks: what it raises is returned to the Specialist once.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
from app.adk.runtime import Decider, Question
|
||||
from app.workflows.chain import REQUIREMENTS_FILE, read_list
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_TEXT = 200
|
||||
|
||||
|
||||
class RequirementsReview:
|
||||
def __init__(self, decider: Decider, *, act: float = 0.3) -> None:
|
||||
self._decider = decider
|
||||
# Lower than the router's threshold, from measurement on 2026-10-08: across
|
||||
# 61 requirements from three passed runs the highest answer was 0.10, and
|
||||
# requirements written to name a product scored 0.34, 0.61 and 0.93.
|
||||
self._act = act
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
kind: DeliverableKind,
|
||||
attachments: Mapping[str, str],
|
||||
upstream: Mapping[str, Mapping[str, Any]] | None,
|
||||
scope: str = "",
|
||||
) -> Sequence[str]:
|
||||
"""The requirements that name a product, each as a line for the Specialist."""
|
||||
if kind is not DeliverableKind.REQUIREMENTS:
|
||||
return ()
|
||||
items = [
|
||||
item
|
||||
for item in read_list(
|
||||
{"requirements": {"attachments": attachments}},
|
||||
"requirements",
|
||||
REQUIREMENTS_FILE,
|
||||
)
|
||||
if isinstance(item.get("id"), str) and isinstance(item.get("text"), str)
|
||||
]
|
||||
if not items:
|
||||
return ()
|
||||
answers = await self._decider.decide(
|
||||
"Requirements for a workload, written before any product is chosen.",
|
||||
{
|
||||
item["id"]: Question(
|
||||
"noul",
|
||||
f'Requirement: "{item["text"]}". Does it name a specific cloud '
|
||||
"provider's product or managed service, as opposed to "
|
||||
"describing what is needed?",
|
||||
)
|
||||
for item in items
|
||||
},
|
||||
scope=scope,
|
||||
)
|
||||
if not answers:
|
||||
return ()
|
||||
raised = [
|
||||
f'{item["id"]} names a product or service: "{item["text"][:_TEXT]}". '
|
||||
"Say what is needed, not what to use"
|
||||
for item in items
|
||||
if (answer := answers.get(item["id"])) is not None
|
||||
and answer.is_yes(self._act)
|
||||
]
|
||||
if raised:
|
||||
logger.info("[%s] requirements review raised %d", scope, len(raised))
|
||||
return tuple(raised)
|
||||
Reference in New Issue
Block a user