feat: establish the Target architecture as a graph before code

Design is now approved before any infrastructure code is written (ADR-0005).
A new target-architect Specialist produces one graph of Elements,
Relationships and Checks; the diagram is rendered from it, deployment
guidance must account for every Check, and validation gives a verdict per
Check. Requirements, components, products and recommendations carry ids in
data files, and a submission with a gap in the Requirements chain is refused.

Also in this commit:
- Customer cases group Executions, with an intent history per case
- lesson candidates are captured when an Execution ends
- evaluation gains chain, verdict, judged and interactive checks, and
  per-case thresholds
- Terraform providers are cached between checks
- the requirements analyst proposes defaults instead of asking
- the legacy-baseline case and the checks that only read it are removed
- ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed
- plans for iterations 4 to 7 under .agents/plans/

Unproven with a real model: no evaluation run has completed on this design.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-07 13:59:09 +01:00
parent 7ee49ab352
commit e9246828b3
586 changed files with 5166 additions and 22432 deletions

View File

@@ -6,10 +6,12 @@ append outcomes that the next Orchestrator turn applies in order.
from __future__ import annotations
import dataclasses
import json
import logging
import re
import time
from collections.abc import Awaitable, Callable, Mapping, Sequence
from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any
@@ -17,22 +19,32 @@ from typing import Any
from langgraph.types import Command, Send, interrupt
from app.adk.runtime import (
REJECTION_PREFIXES,
AgentRuntime,
AgentTask,
BudgetExhausted,
NeedsInput,
Outcome,
RoleSpec,
RunLimiter,
Stopped,
Submitted,
ToolFactory,
ToolSpec,
TurnEnded,
)
from app.customer_case import close_intent
from app.recorder import ExecutionRecorder
from app.states.execution_state import ExecutionState
from app.states.execution_state import (
UNASSIGNED_CASE,
ExecutionState,
execution_dir,
lessons_file,
)
from app.states.ledger import Ledger, LedgerRefusal
from app.states.lessons import collect_lesson_candidates, refusal
from app.tools.execution import ORCHESTRATOR, OrchestratorToolbox, specialist_tools
from app.workflows.chain import VERDICTS_FILE, validation_outcome
from app.workflows.deliverables import (
DEPENDENCY_GRAPH,
PHASE_ORDER,
@@ -47,9 +59,12 @@ NO_USER_REPLY = (
"'Assumptions', and proceed."
)
GUIDE_FILENAME = "solution-architecture-guide.md"
MANIFEST_FILENAME = "execution.json" # request, status and files of one Execution
def _no_tools(role: RoleSpec, deliverable: str | None) -> Sequence[ToolSpec]:
def _no_tools(
role: RoleSpec, deliverable: str | None, working: MutableMapping[str, Any]
) -> Sequence[ToolSpec]:
return ()
@@ -73,9 +88,42 @@ def make_execution_nodes(
output_root: Path,
max_orchestrator_turns: int = 100,
tool_factory: ToolFactory | None = None,
limiter: RunLimiter | None = None,
skill_versions: Mapping[str, str] | None = None,
) -> ExecutionNodes:
tool_factory = tool_factory or _no_tools
def close(state: ExecutionState, status: str, *, has_files: bool) -> None:
close_intent(
output_root,
_case(state),
execution_id=state["execution_id"],
status=status,
has_files=has_files,
)
async def record_lessons(
state: ExecutionState, ledger: Ledger, status: str
) -> None:
record = collect_lesson_candidates(
execution_id=state["execution_id"],
customer_case=_case(state),
status=status,
ledger=ledger,
outcomes=state.get("outcomes", []),
orchestrator_refusals=state.get("refusals", []),
role_skills={name: role.skills for name, role in roles.items()},
skill_versions=skill_versions or {},
)
# Kept apart from deliverables/: an Execution that ends with warnings writes
# no Deliverable files, but what went wrong in it is still worth keeping.
path = lessons_file(output_root, state["execution_id"], _case(state))
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(record, indent=2, ensure_ascii=False), encoding="utf-8"
)
await recorder.record_lesson_candidates(state["execution_id"], record)
async def record_progress(
state: ExecutionState, ledger: Ledger, status: str | None = None
) -> int:
@@ -102,6 +150,7 @@ def make_execution_nodes(
ledger.revisions(kind) for kind in DEPENDENCY_GRAPH
),
"warnings": state.get("warnings", []),
"customer_case": _case(state),
},
)
return len(ledger.events)
@@ -113,6 +162,7 @@ def make_execution_nodes(
notes = list(state.get("notes", []))
pending_permission = state.get("pending_permission")
warnings = list(state.get("warnings", []))
drafts = dict(state.get("drafts", {}))
outcomes = state.get("outcomes", [])
for record in outcomes[state.get("outcomes_applied", 0) :]:
@@ -124,6 +174,7 @@ def make_execution_nodes(
"content": record["content"],
"attachments": record["attachments"],
}
drafts.pop(kind.value, None)
await recorder.record_submission(
state["execution_id"],
kind.value,
@@ -143,6 +194,11 @@ def make_execution_nodes(
"requested_by": [*requested, kind.value],
}
continue
if record.get("draft"):
drafts[kind.value] = {
"content": record["draft"],
"attachments": record.get("draft_attachments", {}),
}
questions = record.get("questions") or [
(
f"The {record['role']} stopped: {record['summary']} "
@@ -154,6 +210,7 @@ def make_execution_nodes(
turns = state.get("turns", 0) + 1
update: dict[str, Any] = {
"deliverables": deliverables,
"drafts": drafts,
"pending_questions": pending_questions,
"pending_permission": pending_permission,
"outcomes_applied": len(outcomes),
@@ -180,13 +237,21 @@ def make_execution_nodes(
return finish_with("end_with_warnings", warnings=warnings)
toolbox = OrchestratorToolbox(ledger, deliverables, pending_questions)
refusals = list(state.get("refusals", []))
update["refusals"] = refusals
task = AgentTask(
role=ORCHESTRATOR,
prompt="Decide the next step for this Execution using your tools.",
context=_orchestrator_context(state, ledger, pending_questions, notes),
context=_orchestrator_context(
state, ledger, deliverables, pending_questions, notes
),
)
started = time.monotonic()
outcome = await runtime.run(roles[ORCHESTRATOR], task, toolbox.tools())
outcome = await runtime.run(
roles[ORCHESTRATOR],
task,
_noting_refusals(toolbox.tools(), refusals, ORCHESTRATOR, None),
)
logger.info(
"[%s] orchestrator turn %d (%s phase) ended in %.0fs: %s",
state["execution_id"],
@@ -209,7 +274,9 @@ def make_execution_nodes(
sends = [
Send(
"specialist_task",
_specialist_input(state, ledger, deliverables, kind, attempt),
_specialist_input(
state, ledger, deliverables, drafts, kind, attempt
),
)
for kind, attempt in toolbox.started.items()
]
@@ -257,10 +324,18 @@ def make_execution_nodes(
async def specialist_task(task_input: dict[str, Any]) -> dict[str, Any]:
role = roles[task_input["role"]]
kind = DeliverableKind(task_input["kind"])
tools = specialist_tools(
kind,
extra=tool_factory(role, kind.value),
allow_cloud_read=task_input["allow_cloud_read"],
working: dict[str, Any] = dict(task_input.get("working") or {})
refusals: list[dict[str, Any]] = []
tools = _noting_refusals(
specialist_tools(
kind,
extra=tool_factory(role, kind.value, working),
allow_cloud_read=task_input["allow_cloud_read"],
upstream=task_input["context"].get("dependencies"),
),
refusals,
role.name,
kind.value,
)
task = AgentTask(
role=role.name,
@@ -277,7 +352,11 @@ def make_execution_nodes(
)
started = time.monotonic()
try:
outcome: Outcome = await runtime.run(role, task, tools)
if limiter:
async with limiter.slot():
outcome: Outcome = await runtime.run(role, task, tools)
else:
outcome = await runtime.run(role, task, tools)
except Exception as exc:
logger.exception("Specialist %s failed on %s", role.name, kind)
outcome = Stopped(f"it failed with {type(exc).__name__}: {exc}")
@@ -291,14 +370,18 @@ def make_execution_nodes(
)
return {
"outcomes": [
_outcome_record(role.name, kind, task_input["attempt"], outcome)
]
{
**_outcome_record(role.name, kind, task_input["attempt"], outcome),
"refusals": refusals,
}
],
"working": {kind.value: working},
}
async def await_user(state: ExecutionState) -> Command:
prompt = state["prompt"] or {}
if state.get("pre_approved"):
reply = _automatic_reply(prompt, state.get("scripted_answers", []))
reply = automatic_reply(prompt, state.get("scripted_answers", []))
else:
reply = interrupt(prompt)
@@ -359,16 +442,17 @@ def make_execution_nodes(
async def finalize(state: ExecutionState) -> dict[str, Any]:
ledger = Ledger.from_dict(state["ledger"])
_write_outputs(
output_root / "deliverables" / "executions" / state["execution_id"],
state.get("deliverables", {}),
)
_write_outputs(output_root, state)
close(state, "completed", has_files=True)
await record_progress(state, ledger, "completed")
await record_lessons(state, ledger, "completed")
return {"status": "completed"}
async def end_with_warnings(state: ExecutionState) -> dict[str, Any]:
ledger = Ledger.from_dict(state["ledger"])
close(state, "completed_with_warnings", has_files=False)
await record_progress(state, ledger, "completed_with_warnings")
await record_lessons(state, ledger, "completed_with_warnings")
return {"status": "completed_with_warnings"}
return ExecutionNodes(
@@ -383,6 +467,28 @@ def make_execution_nodes(
# --- helpers ---------------------------------------------------------------------------
def _noting_refusals(
tools: Sequence[ToolSpec],
sink: list[dict[str, Any]],
role: str,
deliverable: str | None,
) -> list[ToolSpec]:
"""The same tools, noting each reply that means the call did nothing."""
def noting(tool: ToolSpec) -> ToolSpec:
async def handler(**kwargs: Any) -> Any:
result = await tool.handler(**kwargs)
if result.outcome is None and result.message.startswith(REJECTION_PREFIXES):
sink.append(
refusal(role, deliverable, tool.name, tool.tier, result.message)
)
return result
return dataclasses.replace(tool, handler=handler)
return [noting(tool) for tool in tools]
def _summarise_outcome(outcome: Outcome) -> str:
if isinstance(outcome, TurnEnded):
return f"called {outcome.tool}"
@@ -405,10 +511,19 @@ def _current_phase(ledger: Ledger) -> str:
def _orchestrator_context(
state: ExecutionState,
ledger: Ledger,
deliverables: Mapping[str, Mapping[str, Any]],
pending_questions: Sequence[Mapping[str, Any]],
notes: Sequence[str],
) -> dict[str, Any]:
validation = deliverables.get(DeliverableKind.VALIDATION_RESULTS.value)
return {
# The verdict on each Check, once validation is submitted: failed Checks are
# faults in the code; the rest are gaps in the Target architecture.
"validation": validation_outcome(
validation.get("attachments", {}).get(VERDICTS_FILE, "")
)
if validation
else None,
"workflow_request": state["workflow_request"],
"pre_approved": bool(state.get("pre_approved")),
"allow_cloud_read": state.get("allow_cloud_read"),
@@ -438,6 +553,7 @@ def _specialist_input(
state: ExecutionState,
ledger: Ledger,
deliverables: Mapping[str, Mapping[str, Any]],
drafts: Mapping[str, Mapping[str, Any]],
kind: DeliverableKind,
attempt: int,
) -> dict[str, Any]:
@@ -448,15 +564,18 @@ def _specialist_input(
"kind": kind.value,
"attempt": attempt,
"allow_cloud_read": state.get("allow_cloud_read"),
"working": dict(state.get("working", {}).get(kind.value, {})),
"context": {
"workflow_request": state["workflow_request"],
"pre_approved": bool(state.get("pre_approved")),
"contract": spec.contract(),
"dependencies": {
dependency.value: dict(deliverables[dependency.value])
for dependency in inputs_of(kind)
},
"feedback": list(ledger.feedback(kind)),
"answers": list(state.get("answers", {}).get(kind.value, [])),
"draft": dict(drafts[kind.value]) if kind.value in drafts else None,
"attempt": attempt,
},
}
@@ -479,6 +598,8 @@ def _outcome_record(
"type": "needs_input",
"questions": list(outcome.questions),
"permission": outcome.permission,
"draft": outcome.draft,
"draft_attachments": dict(outcome.draft_attachments),
}
if isinstance(outcome, BudgetExhausted):
return {**base, "type": "budget_exhausted", "summary": outcome.summary}
@@ -495,7 +616,7 @@ def _describe_unfinished(outcome: Outcome) -> str:
return f"Ended with an unexpected outcome: {type(outcome).__name__}."
def _automatic_reply(
def automatic_reply(
prompt: Mapping[str, Any], scripted_answers: Sequence[Sequence[str]] = ()
) -> dict[str, Any]:
"""The reply in a pre-approved Execution: approve, deny cloud reads, and answer
@@ -543,18 +664,44 @@ def _render_prompt(prompt: Mapping[str, Any]) -> str:
return "\n".join(item["question"] for item in prompt.get("questions", []))
def _write_outputs(
out_dir: Path, deliverables: Mapping[str, Mapping[str, Any]]
) -> None:
out_dir.mkdir(parents=True, exist_ok=True)
def _case(state: ExecutionState) -> str:
return state.get("customer_case") or UNASSIGNED_CASE
def _write_outputs(output_root: Path, state: ExecutionState) -> None:
"""Write a finished Execution's files under its Customer case, with a manifest
that ties the request to what was generated for it."""
customer_case = _case(state)
execution_id = state["execution_id"]
out_dir = execution_dir(output_root, execution_id, customer_case)
deliverables: Mapping[str, Mapping[str, Any]] = state.get("deliverables", {})
files: dict[str, str] = {}
by_kind: dict[str, dict[str, Any]] = {}
guide = deliverables.get(DeliverableKind.SOLUTION_GUIDE.value)
if guide:
(out_dir / GUIDE_FILENAME).write_text(guide["content"], encoding="utf-8")
files[GUIDE_FILENAME] = guide["content"]
for kind, deliverable in deliverables.items():
path = out_dir / "deliverables" / f"{kind}.md"
document = f"deliverables/{kind}.md"
files[document] = deliverable["content"]
attachments = dict(deliverable.get("attachments", {}))
files.update(attachments)
by_kind[kind] = {"document": document, "attachments": sorted(attachments)}
for name, content in files.items():
path = out_dir / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(deliverable["content"], encoding="utf-8")
for name, content in deliverable.get("attachments", {}).items():
attachment = out_dir / name
attachment.parent.mkdir(parents=True, exist_ok=True)
attachment.write_text(content, encoding="utf-8")
path.write_text(content, encoding="utf-8")
manifest = {
"customer_case": customer_case,
"execution_id": execution_id,
"session_id": state.get("session_id"),
"intent_version": state.get("intent_version"),
"workflow_request": state["workflow_request"],
"status": "completed",
"solution_guide": GUIDE_FILENAME if guide else None,
"deliverables": by_kind,
}
out_dir.mkdir(parents=True, exist_ok=True)
(out_dir / MANIFEST_FILENAME).write_text(
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
)