feat: establish the Target architecture as a graph before code
Design is now approved before any infrastructure code is written (ADR-0005). A new target-architect Specialist produces one graph of Elements, Relationships and Checks; the diagram is rendered from it, deployment guidance must account for every Check, and validation gives a verdict per Check. Requirements, components, products and recommendations carry ids in data files, and a submission with a gap in the Requirements chain is refused. Also in this commit: - Customer cases group Executions, with an intent history per case - lesson candidates are captured when an Execution ends - evaluation gains chain, verdict, judged and interactive checks, and per-case thresholds - Terraform providers are cached between checks - the requirements analyst proposes defaults instead of asking - the legacy-baseline case and the checks that only read it are removed - ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed - plans for iterations 4 to 7 under .agents/plans/ Unproven with a real model: no evaluation run has completed on this design. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -6,10 +6,12 @@ append outcomes that the next Orchestrator turn applies in order.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -17,22 +19,32 @@ from typing import Any
|
||||
from langgraph.types import Command, Send, interrupt
|
||||
|
||||
from app.adk.runtime import (
|
||||
REJECTION_PREFIXES,
|
||||
AgentRuntime,
|
||||
AgentTask,
|
||||
BudgetExhausted,
|
||||
NeedsInput,
|
||||
Outcome,
|
||||
RoleSpec,
|
||||
RunLimiter,
|
||||
Stopped,
|
||||
Submitted,
|
||||
ToolFactory,
|
||||
ToolSpec,
|
||||
TurnEnded,
|
||||
)
|
||||
from app.customer_case import close_intent
|
||||
from app.recorder import ExecutionRecorder
|
||||
from app.states.execution_state import ExecutionState
|
||||
from app.states.execution_state import (
|
||||
UNASSIGNED_CASE,
|
||||
ExecutionState,
|
||||
execution_dir,
|
||||
lessons_file,
|
||||
)
|
||||
from app.states.ledger import Ledger, LedgerRefusal
|
||||
from app.states.lessons import collect_lesson_candidates, refusal
|
||||
from app.tools.execution import ORCHESTRATOR, OrchestratorToolbox, specialist_tools
|
||||
from app.workflows.chain import VERDICTS_FILE, validation_outcome
|
||||
from app.workflows.deliverables import (
|
||||
DEPENDENCY_GRAPH,
|
||||
PHASE_ORDER,
|
||||
@@ -47,9 +59,12 @@ NO_USER_REPLY = (
|
||||
"'Assumptions', and proceed."
|
||||
)
|
||||
GUIDE_FILENAME = "solution-architecture-guide.md"
|
||||
MANIFEST_FILENAME = "execution.json" # request, status and files of one Execution
|
||||
|
||||
|
||||
def _no_tools(role: RoleSpec, deliverable: str | None) -> Sequence[ToolSpec]:
|
||||
def _no_tools(
|
||||
role: RoleSpec, deliverable: str | None, working: MutableMapping[str, Any]
|
||||
) -> Sequence[ToolSpec]:
|
||||
return ()
|
||||
|
||||
|
||||
@@ -73,9 +88,42 @@ def make_execution_nodes(
|
||||
output_root: Path,
|
||||
max_orchestrator_turns: int = 100,
|
||||
tool_factory: ToolFactory | None = None,
|
||||
limiter: RunLimiter | None = None,
|
||||
skill_versions: Mapping[str, str] | None = None,
|
||||
) -> ExecutionNodes:
|
||||
tool_factory = tool_factory or _no_tools
|
||||
|
||||
def close(state: ExecutionState, status: str, *, has_files: bool) -> None:
|
||||
close_intent(
|
||||
output_root,
|
||||
_case(state),
|
||||
execution_id=state["execution_id"],
|
||||
status=status,
|
||||
has_files=has_files,
|
||||
)
|
||||
|
||||
async def record_lessons(
|
||||
state: ExecutionState, ledger: Ledger, status: str
|
||||
) -> None:
|
||||
record = collect_lesson_candidates(
|
||||
execution_id=state["execution_id"],
|
||||
customer_case=_case(state),
|
||||
status=status,
|
||||
ledger=ledger,
|
||||
outcomes=state.get("outcomes", []),
|
||||
orchestrator_refusals=state.get("refusals", []),
|
||||
role_skills={name: role.skills for name, role in roles.items()},
|
||||
skill_versions=skill_versions or {},
|
||||
)
|
||||
# Kept apart from deliverables/: an Execution that ends with warnings writes
|
||||
# no Deliverable files, but what went wrong in it is still worth keeping.
|
||||
path = lessons_file(output_root, state["execution_id"], _case(state))
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(record, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
await recorder.record_lesson_candidates(state["execution_id"], record)
|
||||
|
||||
async def record_progress(
|
||||
state: ExecutionState, ledger: Ledger, status: str | None = None
|
||||
) -> int:
|
||||
@@ -102,6 +150,7 @@ def make_execution_nodes(
|
||||
ledger.revisions(kind) for kind in DEPENDENCY_GRAPH
|
||||
),
|
||||
"warnings": state.get("warnings", []),
|
||||
"customer_case": _case(state),
|
||||
},
|
||||
)
|
||||
return len(ledger.events)
|
||||
@@ -113,6 +162,7 @@ def make_execution_nodes(
|
||||
notes = list(state.get("notes", []))
|
||||
pending_permission = state.get("pending_permission")
|
||||
warnings = list(state.get("warnings", []))
|
||||
drafts = dict(state.get("drafts", {}))
|
||||
|
||||
outcomes = state.get("outcomes", [])
|
||||
for record in outcomes[state.get("outcomes_applied", 0) :]:
|
||||
@@ -124,6 +174,7 @@ def make_execution_nodes(
|
||||
"content": record["content"],
|
||||
"attachments": record["attachments"],
|
||||
}
|
||||
drafts.pop(kind.value, None)
|
||||
await recorder.record_submission(
|
||||
state["execution_id"],
|
||||
kind.value,
|
||||
@@ -143,6 +194,11 @@ def make_execution_nodes(
|
||||
"requested_by": [*requested, kind.value],
|
||||
}
|
||||
continue
|
||||
if record.get("draft"):
|
||||
drafts[kind.value] = {
|
||||
"content": record["draft"],
|
||||
"attachments": record.get("draft_attachments", {}),
|
||||
}
|
||||
questions = record.get("questions") or [
|
||||
(
|
||||
f"The {record['role']} stopped: {record['summary']} "
|
||||
@@ -154,6 +210,7 @@ def make_execution_nodes(
|
||||
turns = state.get("turns", 0) + 1
|
||||
update: dict[str, Any] = {
|
||||
"deliverables": deliverables,
|
||||
"drafts": drafts,
|
||||
"pending_questions": pending_questions,
|
||||
"pending_permission": pending_permission,
|
||||
"outcomes_applied": len(outcomes),
|
||||
@@ -180,13 +237,21 @@ def make_execution_nodes(
|
||||
return finish_with("end_with_warnings", warnings=warnings)
|
||||
|
||||
toolbox = OrchestratorToolbox(ledger, deliverables, pending_questions)
|
||||
refusals = list(state.get("refusals", []))
|
||||
update["refusals"] = refusals
|
||||
task = AgentTask(
|
||||
role=ORCHESTRATOR,
|
||||
prompt="Decide the next step for this Execution using your tools.",
|
||||
context=_orchestrator_context(state, ledger, pending_questions, notes),
|
||||
context=_orchestrator_context(
|
||||
state, ledger, deliverables, pending_questions, notes
|
||||
),
|
||||
)
|
||||
started = time.monotonic()
|
||||
outcome = await runtime.run(roles[ORCHESTRATOR], task, toolbox.tools())
|
||||
outcome = await runtime.run(
|
||||
roles[ORCHESTRATOR],
|
||||
task,
|
||||
_noting_refusals(toolbox.tools(), refusals, ORCHESTRATOR, None),
|
||||
)
|
||||
logger.info(
|
||||
"[%s] orchestrator turn %d (%s phase) ended in %.0fs: %s",
|
||||
state["execution_id"],
|
||||
@@ -209,7 +274,9 @@ def make_execution_nodes(
|
||||
sends = [
|
||||
Send(
|
||||
"specialist_task",
|
||||
_specialist_input(state, ledger, deliverables, kind, attempt),
|
||||
_specialist_input(
|
||||
state, ledger, deliverables, drafts, kind, attempt
|
||||
),
|
||||
)
|
||||
for kind, attempt in toolbox.started.items()
|
||||
]
|
||||
@@ -257,10 +324,18 @@ def make_execution_nodes(
|
||||
async def specialist_task(task_input: dict[str, Any]) -> dict[str, Any]:
|
||||
role = roles[task_input["role"]]
|
||||
kind = DeliverableKind(task_input["kind"])
|
||||
tools = specialist_tools(
|
||||
kind,
|
||||
extra=tool_factory(role, kind.value),
|
||||
allow_cloud_read=task_input["allow_cloud_read"],
|
||||
working: dict[str, Any] = dict(task_input.get("working") or {})
|
||||
refusals: list[dict[str, Any]] = []
|
||||
tools = _noting_refusals(
|
||||
specialist_tools(
|
||||
kind,
|
||||
extra=tool_factory(role, kind.value, working),
|
||||
allow_cloud_read=task_input["allow_cloud_read"],
|
||||
upstream=task_input["context"].get("dependencies"),
|
||||
),
|
||||
refusals,
|
||||
role.name,
|
||||
kind.value,
|
||||
)
|
||||
task = AgentTask(
|
||||
role=role.name,
|
||||
@@ -277,7 +352,11 @@ def make_execution_nodes(
|
||||
)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
outcome: Outcome = await runtime.run(role, task, tools)
|
||||
if limiter:
|
||||
async with limiter.slot():
|
||||
outcome: Outcome = await runtime.run(role, task, tools)
|
||||
else:
|
||||
outcome = await runtime.run(role, task, tools)
|
||||
except Exception as exc:
|
||||
logger.exception("Specialist %s failed on %s", role.name, kind)
|
||||
outcome = Stopped(f"it failed with {type(exc).__name__}: {exc}")
|
||||
@@ -291,14 +370,18 @@ def make_execution_nodes(
|
||||
)
|
||||
return {
|
||||
"outcomes": [
|
||||
_outcome_record(role.name, kind, task_input["attempt"], outcome)
|
||||
]
|
||||
{
|
||||
**_outcome_record(role.name, kind, task_input["attempt"], outcome),
|
||||
"refusals": refusals,
|
||||
}
|
||||
],
|
||||
"working": {kind.value: working},
|
||||
}
|
||||
|
||||
async def await_user(state: ExecutionState) -> Command:
|
||||
prompt = state["prompt"] or {}
|
||||
if state.get("pre_approved"):
|
||||
reply = _automatic_reply(prompt, state.get("scripted_answers", []))
|
||||
reply = automatic_reply(prompt, state.get("scripted_answers", []))
|
||||
else:
|
||||
reply = interrupt(prompt)
|
||||
|
||||
@@ -359,16 +442,17 @@ def make_execution_nodes(
|
||||
|
||||
async def finalize(state: ExecutionState) -> dict[str, Any]:
|
||||
ledger = Ledger.from_dict(state["ledger"])
|
||||
_write_outputs(
|
||||
output_root / "deliverables" / "executions" / state["execution_id"],
|
||||
state.get("deliverables", {}),
|
||||
)
|
||||
_write_outputs(output_root, state)
|
||||
close(state, "completed", has_files=True)
|
||||
await record_progress(state, ledger, "completed")
|
||||
await record_lessons(state, ledger, "completed")
|
||||
return {"status": "completed"}
|
||||
|
||||
async def end_with_warnings(state: ExecutionState) -> dict[str, Any]:
|
||||
ledger = Ledger.from_dict(state["ledger"])
|
||||
close(state, "completed_with_warnings", has_files=False)
|
||||
await record_progress(state, ledger, "completed_with_warnings")
|
||||
await record_lessons(state, ledger, "completed_with_warnings")
|
||||
return {"status": "completed_with_warnings"}
|
||||
|
||||
return ExecutionNodes(
|
||||
@@ -383,6 +467,28 @@ def make_execution_nodes(
|
||||
# --- helpers ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _noting_refusals(
|
||||
tools: Sequence[ToolSpec],
|
||||
sink: list[dict[str, Any]],
|
||||
role: str,
|
||||
deliverable: str | None,
|
||||
) -> list[ToolSpec]:
|
||||
"""The same tools, noting each reply that means the call did nothing."""
|
||||
|
||||
def noting(tool: ToolSpec) -> ToolSpec:
|
||||
async def handler(**kwargs: Any) -> Any:
|
||||
result = await tool.handler(**kwargs)
|
||||
if result.outcome is None and result.message.startswith(REJECTION_PREFIXES):
|
||||
sink.append(
|
||||
refusal(role, deliverable, tool.name, tool.tier, result.message)
|
||||
)
|
||||
return result
|
||||
|
||||
return dataclasses.replace(tool, handler=handler)
|
||||
|
||||
return [noting(tool) for tool in tools]
|
||||
|
||||
|
||||
def _summarise_outcome(outcome: Outcome) -> str:
|
||||
if isinstance(outcome, TurnEnded):
|
||||
return f"called {outcome.tool}"
|
||||
@@ -405,10 +511,19 @@ def _current_phase(ledger: Ledger) -> str:
|
||||
def _orchestrator_context(
|
||||
state: ExecutionState,
|
||||
ledger: Ledger,
|
||||
deliverables: Mapping[str, Mapping[str, Any]],
|
||||
pending_questions: Sequence[Mapping[str, Any]],
|
||||
notes: Sequence[str],
|
||||
) -> dict[str, Any]:
|
||||
validation = deliverables.get(DeliverableKind.VALIDATION_RESULTS.value)
|
||||
return {
|
||||
# The verdict on each Check, once validation is submitted: failed Checks are
|
||||
# faults in the code; the rest are gaps in the Target architecture.
|
||||
"validation": validation_outcome(
|
||||
validation.get("attachments", {}).get(VERDICTS_FILE, "")
|
||||
)
|
||||
if validation
|
||||
else None,
|
||||
"workflow_request": state["workflow_request"],
|
||||
"pre_approved": bool(state.get("pre_approved")),
|
||||
"allow_cloud_read": state.get("allow_cloud_read"),
|
||||
@@ -438,6 +553,7 @@ def _specialist_input(
|
||||
state: ExecutionState,
|
||||
ledger: Ledger,
|
||||
deliverables: Mapping[str, Mapping[str, Any]],
|
||||
drafts: Mapping[str, Mapping[str, Any]],
|
||||
kind: DeliverableKind,
|
||||
attempt: int,
|
||||
) -> dict[str, Any]:
|
||||
@@ -448,15 +564,18 @@ def _specialist_input(
|
||||
"kind": kind.value,
|
||||
"attempt": attempt,
|
||||
"allow_cloud_read": state.get("allow_cloud_read"),
|
||||
"working": dict(state.get("working", {}).get(kind.value, {})),
|
||||
"context": {
|
||||
"workflow_request": state["workflow_request"],
|
||||
"pre_approved": bool(state.get("pre_approved")),
|
||||
"contract": spec.contract(),
|
||||
"dependencies": {
|
||||
dependency.value: dict(deliverables[dependency.value])
|
||||
for dependency in inputs_of(kind)
|
||||
},
|
||||
"feedback": list(ledger.feedback(kind)),
|
||||
"answers": list(state.get("answers", {}).get(kind.value, [])),
|
||||
"draft": dict(drafts[kind.value]) if kind.value in drafts else None,
|
||||
"attempt": attempt,
|
||||
},
|
||||
}
|
||||
@@ -479,6 +598,8 @@ def _outcome_record(
|
||||
"type": "needs_input",
|
||||
"questions": list(outcome.questions),
|
||||
"permission": outcome.permission,
|
||||
"draft": outcome.draft,
|
||||
"draft_attachments": dict(outcome.draft_attachments),
|
||||
}
|
||||
if isinstance(outcome, BudgetExhausted):
|
||||
return {**base, "type": "budget_exhausted", "summary": outcome.summary}
|
||||
@@ -495,7 +616,7 @@ def _describe_unfinished(outcome: Outcome) -> str:
|
||||
return f"Ended with an unexpected outcome: {type(outcome).__name__}."
|
||||
|
||||
|
||||
def _automatic_reply(
|
||||
def automatic_reply(
|
||||
prompt: Mapping[str, Any], scripted_answers: Sequence[Sequence[str]] = ()
|
||||
) -> dict[str, Any]:
|
||||
"""The reply in a pre-approved Execution: approve, deny cloud reads, and answer
|
||||
@@ -543,18 +664,44 @@ def _render_prompt(prompt: Mapping[str, Any]) -> str:
|
||||
return "\n".join(item["question"] for item in prompt.get("questions", []))
|
||||
|
||||
|
||||
def _write_outputs(
|
||||
out_dir: Path, deliverables: Mapping[str, Mapping[str, Any]]
|
||||
) -> None:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
def _case(state: ExecutionState) -> str:
|
||||
return state.get("customer_case") or UNASSIGNED_CASE
|
||||
|
||||
|
||||
def _write_outputs(output_root: Path, state: ExecutionState) -> None:
|
||||
"""Write a finished Execution's files under its Customer case, with a manifest
|
||||
that ties the request to what was generated for it."""
|
||||
customer_case = _case(state)
|
||||
execution_id = state["execution_id"]
|
||||
out_dir = execution_dir(output_root, execution_id, customer_case)
|
||||
deliverables: Mapping[str, Mapping[str, Any]] = state.get("deliverables", {})
|
||||
files: dict[str, str] = {}
|
||||
by_kind: dict[str, dict[str, Any]] = {}
|
||||
guide = deliverables.get(DeliverableKind.SOLUTION_GUIDE.value)
|
||||
if guide:
|
||||
(out_dir / GUIDE_FILENAME).write_text(guide["content"], encoding="utf-8")
|
||||
files[GUIDE_FILENAME] = guide["content"]
|
||||
for kind, deliverable in deliverables.items():
|
||||
path = out_dir / "deliverables" / f"{kind}.md"
|
||||
document = f"deliverables/{kind}.md"
|
||||
files[document] = deliverable["content"]
|
||||
attachments = dict(deliverable.get("attachments", {}))
|
||||
files.update(attachments)
|
||||
by_kind[kind] = {"document": document, "attachments": sorted(attachments)}
|
||||
for name, content in files.items():
|
||||
path = out_dir / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(deliverable["content"], encoding="utf-8")
|
||||
for name, content in deliverable.get("attachments", {}).items():
|
||||
attachment = out_dir / name
|
||||
attachment.parent.mkdir(parents=True, exist_ok=True)
|
||||
attachment.write_text(content, encoding="utf-8")
|
||||
path.write_text(content, encoding="utf-8")
|
||||
|
||||
manifest = {
|
||||
"customer_case": customer_case,
|
||||
"execution_id": execution_id,
|
||||
"session_id": state.get("session_id"),
|
||||
"intent_version": state.get("intent_version"),
|
||||
"workflow_request": state["workflow_request"],
|
||||
"status": "completed",
|
||||
"solution_guide": GUIDE_FILENAME if guide else None,
|
||||
"deliverables": by_kind,
|
||||
}
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
(out_dir / MANIFEST_FILENAME).write_text(
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user