feat: establish the Target architecture as a graph before code

Design is now approved before any infrastructure code is written (ADR-0005).
A new target-architect Specialist produces one graph of Elements,
Relationships and Checks; the diagram is rendered from it, deployment
guidance must account for every Check, and validation gives a verdict per
Check. Requirements, components, products and recommendations carry ids in
data files, and a submission with a gap in the Requirements chain is refused.

Also in this commit:
- Customer cases group Executions, with an intent history per case
- lesson candidates are captured when an Execution ends
- evaluation gains chain, verdict, judged and interactive checks, and
  per-case thresholds
- Terraform providers are cached between checks
- the requirements analyst proposes defaults instead of asking
- the legacy-baseline case and the checks that only read it are removed
- ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed
- plans for iterations 4 to 7 under .agents/plans/

Unproven with a real model: no evaluation run has completed on this design.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-07 13:59:09 +01:00
parent 7ee49ab352
commit e9246828b3
586 changed files with 5166 additions and 22432 deletions

View File

@@ -8,7 +8,7 @@ import pytest
from langgraph.checkpoint.memory import InMemorySaver
from starlette.testclient import TestClient
from app.adk.fake_runtime import Call, FakeAgentRuntime
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.roles import load_roles
from app.agent import Assembly, ExecutionEngine
from app.config import get_settings
@@ -32,17 +32,7 @@ LEGACY_GENERATE_KEYS = {
def submission(kind: DeliverableKind) -> list[Call]:
spec = DEPENDENCY_GRAPH[kind]
body = "\n".join(f"## {section}\nDetails." for section in spec.sections)
return [
Call(
"submit_deliverable",
{
"content": f"# {kind}\n{body}\n",
"attachments": {name: f"# {name}\n" for name in spec.attachments},
},
)
]
return [sample_submission(kind)]
def complete_run_turns() -> dict:
@@ -57,9 +47,16 @@ def complete_run_turns() -> dict:
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(PhaseId.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
[delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)],
[approve(PhaseId.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
[
approve(PhaseId.SOLUTION_ARCHITECTURE),
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
],
[
approve(PhaseId.SOLUTION_IMPLEMENTATION),
delegate(K.VALIDATION_RESULTS),
],
[approve(PhaseId.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
[approve(PhaseId.PACKAGING), Call("finish")],
],
@@ -272,7 +269,14 @@ def test_generate_runs_pre_approved_and_keeps_the_legacy_response_keys(tmp_path)
assert body["artifacts"]["terraform_code"] == "# terraform/main.tf\n"
assert body["artifacts"]["source_discovery_doc"] == "Details."
assert "google-cloud-solution-architecture" in body["active_skills"]
assert (tmp_path / "deliverables" / "executions" / body["execution_id"]).is_dir()
assert (
tmp_path
/ "deliverables"
/ "cases"
/ "unassigned"
/ "executions"
/ body["execution_id"]
).is_dir()
@pytest.mark.parametrize("payload", [{}, {"request": " "}])
@@ -291,3 +295,27 @@ def test_health_reports_mcp_discovery_gaps(tmp_path):
assert body["status"] == "healthy"
assert body["mcp_discovery_gaps"] == {"compute": "API not enabled"}
def test_generate_groups_the_execution_under_the_callers_customer_case(tmp_path):
turns = {
"orchestrator": [
[Call("delegate", {"kinds": ["requirements"]})],
[],
],
"requirements-analyst": [submission(DeliverableKind.REQUIREMENTS)],
}
client, _ = client_for(tmp_path, turns)
with client:
refused = client.post(
"/generate", json={"request": "Design ingestion", "customer_case": "../x"}
)
accepted = client.post(
"/generate",
json={"request": "Design ingestion", "customer_case": "acme-webhooks"},
)
assert refused.status_code == 400
assert "customer_case" in refused.json()["error"]
assert accepted.status_code == 200, accepted.json()
assert (tmp_path / "lessons" / "candidates" / "acme-webhooks").is_dir()

View File

@@ -12,6 +12,7 @@ from langgraph.checkpoint.memory import InMemorySaver
from pydantic import Field
from app.adk.adk_runtime import AdkAgentRuntime
from app.adk.fake_runtime import SAMPLE_DATA
from app.adk.roles import load_roles
from app.agent import ExecutionEngine
from app.providers import load_provider
@@ -69,7 +70,16 @@ async def test_engine_runs_role_files_through_adk_with_one_voice(tmp_path):
call(
"needs_input", questions=["What availability target do you need?"]
),
call("submit_deliverable", content=requirements_markdown()),
call(
"submit_deliverable",
content=requirements_markdown(),
attachments=[
{
"path": "requirements.json",
"content": SAMPLE_DATA["requirements.json"],
}
],
),
]
),
}

View File

@@ -33,7 +33,12 @@ def test_roles_take_skills_mcp_servers_and_models_from_the_bundle(bundle):
assert roles[PILLAR_ADVISOR].skills == ()
assert roles["product-selector"].mcp_servers == ("developer_knowledge",)
assert "asset_inventory" in roles["requirements-analyst"].mcp_servers
assert roles["diagrammer"].model == bundle.models.for_role("diagrammer")
assert roles["describer"].model == bundle.models.default
assert roles["target-architect"].skills == (
"google-cloud-solution-architecture",
"requirements-chain",
)
assert roles["target-architect"].mcp_servers == ("developer_knowledge",)
assert roles["orchestrator"].model == bundle.models.for_role("orchestrator")
assert roles["decomposer"].model == bundle.models.default
@@ -41,7 +46,8 @@ def test_roles_take_skills_mcp_servers_and_models_from_the_bundle(bundle):
def test_roles_name_only_the_local_tools_they_need(bundle):
roles = load_roles(AGENTS_DIR, bundle)
assert roles["diagrammer"].tools == ("mermaid_check",)
assert roles["iac-author"].tools == ("terraform_fmt", "terraform_validate")
assert roles["target-architect"].tools == ()
assert roles["validator"].tools == ("terraform_validate", "terraform_plan")
assert roles["requirements-analyst"].tools == ()

View File

@@ -2,8 +2,10 @@
from pathlib import Path
from app.adk.fake_runtime import SAMPLE_DATA
from app.adk.runtime import ToolResult
from app.agent import Interaction
from app.workflows.target_architecture import parse_graph, render_mermaid
from eval.scoring import CaseSpec, score_execution, template_headings
TEMPLATE = (
@@ -25,11 +27,18 @@ def guide(headings=HEADINGS) -> str:
return "\n".join(f"## {heading}\nContent." for heading in headings)
GRAPH = SAMPLE_DATA["target-architecture.json"]
def execution(status="completed", **overrides) -> Interaction:
deliverables = {
"requirements": {
"content": "## Functional requirements\nAccept webhooks.\n## Current state\nNone.",
"attachments": {},
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]},
},
"technical_decomposition": {
"content": "## Components\nIngress.",
"attachments": {"components.json": SAMPLE_DATA["components.json"]},
},
"product_selection": {
"content": (
@@ -39,12 +48,19 @@ def execution(status="completed", **overrides) -> Interaction:
"**Pros**: control **Cons**: toil |\n"
"| Queue | Pub/Sub | ... | Cloud Tasks | ... |\n| Archive | Cloud Storage | ... |"
),
"attachments": {},
"attachments": {"products.json": SAMPLE_DATA["products.json"]},
},
"architecture_diagram": {
"content": "diagram",
"design_recommendations": {
"content": "## Reliability\nUse two zones because NFR-1 asks for 99.9%.",
"attachments": {
"architecture.mmd": "flowchart TD\n A[Client] --> B[Run]\n"
"recommendations.json": SAMPLE_DATA["recommendations.json"]
},
},
"target_architecture": {
"content": "## Elements\n## Relationships\n## Checks\n",
"attachments": {
"target-architecture.json": GRAPH,
"architecture.mmd": render_mermaid(parse_graph(GRAPH)),
},
},
"deployment_guidance": {
@@ -52,8 +68,13 @@ def execution(status="completed", **overrides) -> Interaction:
"attachments": {
"terraform/main.tf": "resource {}",
"terraform/variables.tf": "",
"implementation.json": SAMPLE_DATA["implementation.json"],
},
},
"validation_results": {
"content": "## Deployment dry-run\nPass.",
"attachments": {"verdicts.json": SAMPLE_DATA["verdicts.json"]},
},
"solution_guide": {"content": guide(), "attachments": {}},
}
for kind, value in overrides.items():
@@ -148,7 +169,7 @@ async def test_invalid_terraform_and_diagram_fail_and_warnings_fail_completion()
result = await score_execution(
CASE,
execution("completed_with_warnings", architecture_diagram=diagram),
execution("completed_with_warnings", target_architecture=diagram),
HEADINGS,
failing_terraform,
)
@@ -181,3 +202,276 @@ def test_benchmark_cases_load_with_valid_answer_patterns():
for pattern, answer in case.scripted_answers:
re.compile(pattern)
assert answer
# --- checks that can see quality (iteration 4, step 8) ---------------------------------
class FakeJudge:
"""Scores every question with a fixed value, or per id from a mapping."""
def __init__(self, default=1.0, **by_id):
self.default, self.by_id = default, by_id
self.asked: list[dict] = []
async def answer(self, document, questions):
self.asked.append(dict(questions))
return {key: self.by_id.get(key, self.default) for key in questions}
CHECKLISTS = {
"reliability": ["No single point of failure.", "Backups are tested."],
"security": ["Least privilege is applied."],
}
def deliverable(content: str, **attachments: str) -> dict:
return {"content": content, "attachments": attachments}
async def test_a_requirement_that_reaches_no_check_breaks_the_chain():
import json
requirements = json.loads(SAMPLE_DATA["requirements.json"])
requirements.append({"id": "NFR-2", "kind": "non_functional", "text": "Retain."})
result = await score_execution(
CASE,
execution(
requirements=deliverable(
"r", **{"requirements.json": json.dumps(requirements)}
)
),
HEADINGS,
passing_terraform,
)
chain = check(result, "requirements_chain_complete")
assert not chain.passed
assert chain.detail == "Requirement NFR-2 has no Check"
async def test_a_check_left_failed_or_unverifiable_fails_the_case():
import json
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
verdicts["checks"][0]["verdict"] = "fail"
verdicts["checks"][1]["verdict"] = "not_verifiable"
verdicts["proposed_additions"] = [
{"statement": "s", "applies_to": "E-1", "reason": "NFR-1"}
]
result = await score_execution(
CASE,
execution(
validation_results=deliverable(
"v", **{"verdicts.json": json.dumps(verdicts)}
)
),
HEADINGS,
passing_terraform,
)
assert (
check(result, "every_check_passes").detail == "K-1 failed; K-2 not verifiable"
)
async def test_a_diagram_that_is_not_the_graphs_fails():
drawn = deliverable(
"t",
**{
"target-architecture.json": GRAPH,
"architecture.mmd": "flowchart LR\n a --> b\n",
},
)
result = await score_execution(
CASE, execution(target_architecture=drawn), HEADINGS, passing_terraform
)
assert not check(result, "diagram_matches_graph").passed
assert check(await _complete(), "diagram_matches_graph").passed
async def _complete():
return await score_execution(CASE, execution(), HEADINGS, passing_terraform)
async def test_judged_checks_run_only_with_a_judge():
without = await score_execution(CASE, execution(), HEADINGS, passing_terraform)
judged = await score_execution(
CASE,
execution(),
HEADINGS,
passing_terraform,
judge=FakeJudge(),
pillar_checklists=CHECKLISTS,
)
names = {c.name for c in judged.checks} - {c.name for c in without.checks}
assert names == {"pillar_checklists_covered", "recommendations_quality"}
assert judged.score == 1.0
async def test_checklist_coverage_is_judged_per_pillar_and_the_weakest_decides():
judge = FakeJudge(**{"reliability.1": 0.0})
result = await score_execution(
CASE,
execution(),
HEADINGS,
passing_terraform,
judge=judge,
pillar_checklists=CHECKLISTS,
)
coverage = check(result, "pillar_checklists_covered")
assert not coverage.passed
assert coverage.detail == "reliability 50%, security 100%"
assert "Backups are tested." in judge.asked[0]["reliability.1"]["text"]
async def test_generic_recommendations_fail_the_quality_rubric():
result = await score_execution(
CASE,
execution(),
HEADINGS,
passing_terraform,
judge=FakeJudge(default=1.0, specific=0.0, actionable=0.33, traceable=0.33),
pillar_checklists=CHECKLISTS,
)
quality = check(result, "recommendations_quality")
assert not quality.passed
assert quality.detail.startswith("mean 0.42")
async def test_an_interactive_case_checks_specialists_ask_before_submitting():
from dataclasses import replace
interactive = replace(CASE, interactive=True)
asked = [
{"deliverable": "requirements", "outcome": "NeedsInput"},
{"deliverable": "requirements", "outcome": "Submitted"},
{"deliverable": None, "outcome": "TurnEnded"},
]
silent = [*asked, {"deliverable": "product_selection", "outcome": "Submitted"}]
good = await score_execution(
interactive, execution(), HEADINGS, passing_terraform, timeline=asked
)
bad = await score_execution(
interactive, execution(), HEADINGS, passing_terraform, timeline=silent
)
batch = await score_execution(CASE, execution(), HEADINGS, passing_terraform)
assert check(good, "specialists_ask_before_submitting").passed
assert (
check(bad, "specialists_ask_before_submitting").detail
== "submitted without asking: product_selection"
)
assert "specialists_ask_before_submitting" not in {c.name for c in batch.checks}
def test_exceeding_a_threshold_is_a_regression():
from dataclasses import replace
from eval.scoring import regressions
case = replace(CASE, thresholds={"revisions": 2, "seconds": 3000, "tokens": 4e6})
assert regressions(case, {"revisions": 2, "seconds": 2999.5, "tokens": 3.9e6}) == []
assert regressions(case, {"revisions": 3, "seconds": 3100, "tokens": None}) == [
"revisions: 3 > 2",
"seconds: 3100 > 3000",
]
assert regressions(CASE, {"revisions": 9}) == []
def test_every_pillar_skill_yields_checklist_items():
from app.providers import load_provider
from eval.scoring import checklist_items
bundle = load_provider("gcp")
for pillar, skill in bundle.pillar_skills.items():
items = checklist_items(bundle.skills[skill].path / "SKILL.md")
assert len(items) >= 3, pillar
assert all(
not item.startswith(("-", "*")) and "\n" not in item for item in items
)
def test_a_judge_reply_is_parsed_defensively():
from eval.judge import parse_scores
questions = {"a": {}, "b": {}, "c": {}, "d": {}}
assert parse_scores('{"a": 1, "b": 0.33, "c": true, "d": 7}', questions) == {
"a": 1.0,
"b": 0.33,
"c": 1.0,
"d": 1.0,
}
assert parse_scores("not json", questions) == dict.fromkeys(questions, 0.0)
assert parse_scores('{"a": "yes"}', questions)["a"] == 0.0
async def test_a_scripted_user_drives_an_interactive_execution(tmp_path):
from dataclasses import replace
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.runtime import RoleSpec
from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
from eval.solution_eval import run_with_scripted_user
requirements = DeliverableKind.REQUIREMENTS
analyst = DEPENDENCY_GRAPH[requirements].specialist
roles = {
"orchestrator": RoleSpec("orchestrator", "", budget=10),
**{
spec.specialist: RoleSpec(spec.specialist, "", budget=10)
for spec in DEPENDENCY_GRAPH.values()
},
}
runtime = FakeAgentRuntime(
{
"orchestrator": [
[Call("delegate", {"kinds": ["requirements"]})],
[Call("ask_user")],
[Call("delegate", {"kinds": ["requirements"]})],
[Call("ask_user", {"questions": ["Which region?"]})],
[Call("ask_user", {"questions": ["Anything else?"]})],
],
analyst: [
[Call("needs_input", {"questions": ["Do you approve this draft?"]})],
[sample_submission(requirements)],
],
}
)
engine = ExecutionEngine(
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
)
case = replace(
CASE,
interactive=True,
scripted_answers=(("approve", "Approved."), ("region", "europe-west2.")),
)
final = await run_with_scripted_user(
engine, case, "exec-interactive", max_user_turns=2
)
replies = [m["text"] for m in final.state["conversation"] if m["from"] == "user"]
assert any("Approved." in reply for reply in replies)
assert any("europe-west2." in reply for reply in replies)
assert final.state["deliverables"]["requirements"]["content"]

View File

@@ -47,6 +47,7 @@ def test_every_phase_groups_at_least_one_deliverable_with_a_distinct_specialist(
assert PHASE_ORDER == (
PhaseId.REQUIREMENTS_DISCOVERY,
PhaseId.SOLUTION_ARCHITECTURE,
PhaseId.SOLUTION_IMPLEMENTATION,
PhaseId.SOLUTION_VALIDATION,
PhaseId.PACKAGING,
)
@@ -56,55 +57,68 @@ def test_every_phase_groups_at_least_one_deliverable_with_a_distinct_specialist(
assert len(specialists) == len(set(specialists))
def test_graph_matches_the_upstream_skill_dependencies():
def test_the_target_architecture_sits_between_the_design_and_the_code():
depends = {kind: set(spec.depends_on) for kind, spec in DEPENDENCY_GRAPH.items()}
K = DeliverableKind
assert depends[K.REQUIREMENTS] == set()
assert depends[K.TECHNICAL_DECOMPOSITION] == {K.REQUIREMENTS}
assert depends[K.PRODUCT_SELECTION] == {K.TECHNICAL_DECOMPOSITION}
assert depends[K.ARCHITECTURE_DIAGRAM] == {K.PRODUCT_SELECTION}
assert depends[K.ARCHITECTURE_DESCRIPTION] == {
K.ARCHITECTURE_DIAGRAM,
K.PRODUCT_SELECTION,
}
assert depends[K.DESIGN_RECOMMENDATIONS] == {K.PRODUCT_SELECTION, K.REQUIREMENTS}
assert depends[K.DEPLOYMENT_GUIDANCE] == {
assert depends[K.TARGET_ARCHITECTURE] == {
K.PRODUCT_SELECTION,
K.DESIGN_RECOMMENDATIONS,
}
# Everything after the graph is derived from it.
assert depends[K.ARCHITECTURE_DESCRIPTION] == {K.TARGET_ARCHITECTURE}
assert depends[K.DEPLOYMENT_GUIDANCE] == {K.TARGET_ARCHITECTURE}
assert depends[K.VALIDATION_RESULTS] == {K.DEPLOYMENT_GUIDANCE}
assert depends[K.SOLUTION_GUIDE] == set(
deliverables_in(PhaseId.SOLUTION_VALIDATION)
)
def test_the_design_is_approved_before_code_is_written():
design = PhaseId.SOLUTION_ARCHITECTURE
implementation = PhaseId.SOLUTION_IMPLEMENTATION
assert DEPENDENCY_GRAPH[DeliverableKind.TARGET_ARCHITECTURE].phase is design
assert DEPENDENCY_GRAPH[DeliverableKind.DEPLOYMENT_GUIDANCE].phase is implementation
assert PHASE_ORDER.index(design) < PHASE_ORDER.index(implementation)
def test_inputs_are_earlier_phases_plus_indirect_dependencies_in_graph_order():
assert inputs_of(DeliverableKind.REQUIREMENTS) == ()
assert inputs_of(DeliverableKind.DEPLOYMENT_GUIDANCE) == (
design = (
DeliverableKind.REQUIREMENTS,
DeliverableKind.TECHNICAL_DECOMPOSITION,
DeliverableKind.PRODUCT_SELECTION,
DeliverableKind.DESIGN_RECOMMENDATIONS,
DeliverableKind.TARGET_ARCHITECTURE,
)
assert inputs_of(DeliverableKind.ARCHITECTURE_DESCRIPTION) == (
DeliverableKind.REQUIREMENTS,
DeliverableKind.TECHNICAL_DECOMPOSITION,
DeliverableKind.PRODUCT_SELECTION,
DeliverableKind.ARCHITECTURE_DIAGRAM,
)
assert inputs_of(DeliverableKind.REQUIREMENTS) == ()
assert inputs_of(DeliverableKind.DEPLOYMENT_GUIDANCE) == design
assert inputs_of(DeliverableKind.ARCHITECTURE_DESCRIPTION) == design
assert inputs_of(DeliverableKind.TARGET_ARCHITECTURE) == design[:4]
assert set(inputs_of(DeliverableKind.SOLUTION_GUIDE)) == set(DeliverableKind) - {
DeliverableKind.SOLUTION_GUIDE
}
def sections_missing(kind: DeliverableKind, content: str) -> list[str]:
return [m for m in check_submission(kind, content, {}) if m.startswith("section:")]
def test_submission_meeting_the_contract_has_nothing_missing():
content = (
"# Deployment guidance\n"
"## 6.1. Deployment prerequisites\nEnable APIs.\n"
"## 6.2. Step-by-step deployment instructions\nRun terraform.\n"
)
attachments = {"terraform/main.tf": "resource {}", "terraform/variables.tf": ""}
attachments = {
"terraform/main.tf": "resource {}",
"terraform/variables.tf": "variable {}",
"implementation.json": "[]",
}
assert (
check_submission(DeliverableKind.DEPLOYMENT_GUIDANCE, content, attachments)
@@ -122,6 +136,7 @@ def test_submission_reports_missing_sections_and_attachments():
assert missing == (
"section: Step-by-step deployment instructions",
"attachment: terraform/variables.tf",
"attachment: implementation.json",
)
@@ -143,7 +158,7 @@ def test_section_matching_ignores_numbering_case_and_trailing_punctuation(headin
"## Sustainability\n"
)
assert check_submission(DeliverableKind.DESIGN_RECOMMENDATIONS, content, {}) == ()
assert sections_missing(DeliverableKind.DESIGN_RECOMMENDATIONS, content) == []
def test_section_matches_a_provider_prefixed_heading_but_not_a_longer_word():
@@ -152,7 +167,7 @@ def test_section_matches_a_provider_prefixed_heading_but_not_a_longer_word():
"## Non-functional requirements\n"
)
assert check_submission(DeliverableKind.PRODUCT_SELECTION, content, {}) == ()
assert sections_missing(DeliverableKind.PRODUCT_SELECTION, content) == []
assert "section: Functional requirements" in check_submission(
DeliverableKind.REQUIREMENTS, content, {}
)
@@ -173,3 +188,32 @@ def test_empty_content_is_refused():
assert check_submission(DeliverableKind.TECHNICAL_DECOMPOSITION, " \n", {}) == (
"content: must not be empty",
)
# --- contracts beyond headings ---------------------------------------------------------
def test_architecture_description_needs_component_and_flow_sections():
missing = check_submission(
DeliverableKind.ARCHITECTURE_DESCRIPTION, "The system works.", {}
)
assert missing == ("section: Components", "section: Flow")
assert (
check_submission(
DeliverableKind.ARCHITECTURE_DESCRIPTION,
"## Architecture components\nA.\n## Data flow\nB.\n",
{},
)
== ()
)
def test_a_required_attachment_must_have_content():
missing = check_submission(
DeliverableKind.VALIDATION_RESULTS,
"## Deployment dry-run\nPass.\n## Architecture and policy analysis\nPass.\n",
{"verdicts.json": " \n"},
)
assert missing == ("attachment: verdicts.json must not be empty",)

View File

@@ -3,13 +3,14 @@
Driven entirely through ExecutionEngine.start/resume with a scripted agent runtime.
"""
import json
from pathlib import Path
import pytest
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from app.adk.fake_runtime import Call, FakeAgentRuntime
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
@@ -50,12 +51,7 @@ def reopen(kind: DeliverableKind, reason: str) -> Call:
FINISH = Call("finish")
def submission(kind: DeliverableKind, note: str = "") -> Call:
spec = DEPENDENCY_GRAPH[kind]
body = "\n".join(f"## {section}\n{note or 'Details.'}" for section in spec.sections)
content = f"# {kind}\n{note or 'Summary.'}\n{body}\n"
attachments = {name: f"# {name}\n" for name in spec.attachments}
return Call("submit_deliverable", {"content": content, "attachments": attachments})
submission = sample_submission
def submits(kind: DeliverableKind) -> list[Call]:
@@ -84,9 +80,16 @@ def happy_path_turns(**extra_specialist_turns) -> dict:
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
[delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)],
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
[
request_approval(P.SOLUTION_ARCHITECTURE),
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
],
[
request_approval(P.SOLUTION_IMPLEMENTATION),
delegate(K.VALIDATION_RESULTS),
],
[request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
[request_approval(P.PACKAGING), FINISH],
],
@@ -112,7 +115,14 @@ async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path
assert set(ledger["phases"].values()) == {"approved"}
assert {d["status"] for d in ledger["deliverables"].values()} == {"submitted"}
out = tmp_path / "deliverables" / "executions" / result.execution_id
out = (
tmp_path
/ "deliverables"
/ "cases"
/ "unassigned"
/ "executions"
/ result.execution_id
)
assert (out / "solution-architecture-guide.md").is_file()
for kind in K:
assert (out / "deliverables" / f"{kind}.md").is_file(), kind
@@ -122,7 +132,7 @@ async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path
assert recorder.statuses[result.execution_id][-1]["status"] == "completed"
assert {s["kind"] for s in recorder.submissions[result.execution_id]} == set(K)
recorded_types = [e.type for e in recorder.events[result.execution_id]]
assert recorded_types.count("approved") == 4
assert recorded_types.count("approved") == 5
async def test_independent_specialists_run_together_between_orchestrator_turns(
@@ -133,12 +143,12 @@ async def test_independent_specialists_run_together_between_orchestrator_turns(
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
roles = [run.role for run in runtime.runs]
fourth_turn = [i for i, role in enumerate(roles) if role == "orchestrator"][3]
assert set(roles[fourth_turn + 1 : fourth_turn + 3]) == {
specialist(K.ARCHITECTURE_DIAGRAM),
specialist(K.DESIGN_RECOMMENDATIONS),
sixth_turn = [i for i, role in enumerate(roles) if role == "orchestrator"][5]
assert set(roles[sixth_turn + 1 : sixth_turn + 3]) == {
specialist(K.ARCHITECTURE_DESCRIPTION),
specialist(K.DEPLOYMENT_GUIDANCE),
}
assert roles[fourth_turn + 3] == "orchestrator"
assert roles[sixth_turn + 3] == "orchestrator"
async def test_specialist_receives_its_dependencies_content(tmp_path):
@@ -154,9 +164,13 @@ async def test_specialist_receives_its_dependencies_content(tmp_path):
"requirements",
"technical_decomposition",
"product_selection",
"architecture_diagram",
"design_recommendations",
"target_architecture",
]
assert "architecture.mmd" in dependencies["architecture_diagram"]["attachments"]
# The engine drew the diagram from the graph when the Target architecture was submitted.
assert dependencies["target_architecture"]["attachments"][
"architecture.mmd"
].startswith("flowchart")
assert describer_run.task.deliverable == "architecture_description"
@@ -334,16 +348,17 @@ async def test_requested_changes_lead_to_a_revision_and_a_new_approval(tmp_path)
async def test_questions_from_parallel_specialists_reach_the_user_in_one_prompt(
tmp_path,
):
def diagrammer_turn(task):
def describer_turn(task):
if task.context["answers"]:
return submits(K.ARCHITECTURE_DIAGRAM)
return [Call("needs_input", {"questions": ["Single region or multi-region?"]})]
return submits(K.ARCHITECTURE_DESCRIPTION)
return [Call("needs_input", {"questions": ["Who reads this description?"]})]
def advisor_turn(task):
def iac_turn(task):
if task.context["answers"]:
return submits(K.DESIGN_RECOMMENDATIONS)
return [Call("needs_input", {"questions": ["What is the RTO?"]})]
return submits(K.DEPLOYMENT_GUIDANCE)
return [Call("needs_input", {"questions": ["Which region?"]})]
both = delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)
runtime = FakeAgentRuntime(
{
"orchestrator": [
@@ -351,47 +366,59 @@ async def test_questions_from_parallel_specialists_reach_the_user_in_one_prompt(
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY)],
[delegate(K.PRODUCT_SELECTION)],
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
[request_approval(P.SOLUTION_ARCHITECTURE)],
[both],
[ask_user()],
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
[both],
[ask_user("Continue?")],
],
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)],
specialist(K.PRODUCT_SELECTION): [submits(K.PRODUCT_SELECTION)],
specialist(K.ARCHITECTURE_DIAGRAM): [diagrammer_turn, diagrammer_turn],
specialist(K.DESIGN_RECOMMENDATIONS): [advisor_turn, advisor_turn],
**{
specialist(kind): [submits(kind)]
for kind in (
K.REQUIREMENTS,
K.TECHNICAL_DECOMPOSITION,
K.PRODUCT_SELECTION,
K.DESIGN_RECOMMENDATIONS,
K.TARGET_ARCHITECTURE,
)
},
specialist(K.ARCHITECTURE_DESCRIPTION): [describer_turn, describer_turn],
specialist(K.DEPLOYMENT_GUIDANCE): [iac_turn, iac_turn],
}
)
engine = make_engine(tmp_path, runtime)
approval = await engine.start("Ingestion")
first = await engine.resume(
design = await engine.resume(
approval.execution_id,
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
)
first = await engine.resume(
design.execution_id,
{"approval": {"phase": "solution_architecture", "decision": "approve"}},
)
assert first.prompt == {
"questions": [
{
"from": "architecture_diagram",
"question": "Single region or multi-region?",
"from": "architecture_description",
"question": "Who reads this description?",
},
{"from": "design_recommendations", "question": "What is the RTO?"},
{"from": "deployment_guidance", "question": "Which region?"},
]
}
second = await engine.resume(
first.execution_id, {"reply": "Single region, RTO 1 hour"}
)
second = await engine.resume(first.execution_id, {"reply": "Engineers, in London"})
deliverables = second.state["ledger"]["deliverables"]
assert deliverables["architecture_diagram"]["status"] == "submitted"
assert deliverables["design_recommendations"]["status"] == "submitted"
advisor_retry = [
r for r in runtime.runs if r.role == specialist(K.DESIGN_RECOMMENDATIONS)
assert deliverables["architecture_description"]["status"] == "submitted"
assert deliverables["deployment_guidance"]["status"] == "submitted"
iac_retry = [
r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)
][1]
assert advisor_retry.task.context["answers"] == [
{"questions": ["What is the RTO?"], "reply": "Single region, RTO 1 hour"}
assert iac_retry.task.context["answers"] == [
{"questions": ["Which region?"], "reply": "Engineers, in London"}
]
@@ -404,14 +431,18 @@ async def test_validation_failure_revises_deployment_guidance_and_reapproves(tmp
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
[delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)],
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
[
request_approval(P.SOLUTION_ARCHITECTURE),
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
],
[request_approval(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[
reopen(K.DEPLOYMENT_GUIDANCE, "terraform plan: missing IAM binding"),
delegate(K.DEPLOYMENT_GUIDANCE),
],
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
[request_approval(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
[request_approval(P.PACKAGING), FINISH],
]
@@ -430,9 +461,10 @@ async def test_validation_failure_revises_deployment_guidance_and_reapproves(tmp
assert result.status == "completed"
events = result.state["ledger"]["events"]
withdrawn = [e["subject"] for e in events if e["type"] == "approval_withdrawn"]
assert withdrawn == ["solution_architecture"]
assert withdrawn == ["solution_implementation"]
requests = [e["subject"] for e in events if e["type"] == "approval_requested"]
assert requests.count("solution_architecture") == 2
assert requests.count("solution_implementation") == 2
assert requests.count("solution_architecture") == 1
iac_runs = [r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)]
assert iac_runs[1].task.context["feedback"] == [
"terraform plan: missing IAM binding"
@@ -456,7 +488,12 @@ async def test_pre_approved_escalation_ends_with_warnings(tmp_path):
assert result.status == "completed_with_warnings"
assert "revision limit" in result.state["warnings"][0]
assert not (tmp_path / "deliverables").exists()
# No Deliverable files are written, but the request stays on record.
case = tmp_path / "deliverables" / "cases" / "unassigned"
assert not (case / "executions").exists()
[intent] = json.loads((case / "intent" / "history.json").read_text())["intents"]
assert intent["status"] == "completed_with_warnings"
assert intent["execution"] is None
# --- budgets -------------------------------------------------------------------------------------
@@ -514,7 +551,7 @@ def cloud_read_factory(calls: list[str]):
tier="cloud_read",
)
def factory(role: RoleSpec, deliverable: str | None):
def factory(role: RoleSpec, deliverable: str | None, working: dict):
return [tool] if role.name == "requirements-analyst" else []
return factory
@@ -686,3 +723,249 @@ async def test_pre_approved_questions_use_matching_scripted_answers(tmp_path):
assert "No user is available" not in first_reply
second_reply = result.state["conversation"][4]["text"]
assert second_reply.startswith("No user is available")
# --- contract and draft in the Specialist's context (iteration 4, steps 2 and 4) -------
async def test_specialist_context_carries_its_contract(tmp_path):
runtime = FakeAgentRuntime(happy_path_turns())
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
run = next(r for r in runtime.runs if r.role == specialist(K.VALIDATION_RESULTS))
assert run.task.context["contract"] == {
"sections": ["Deployment dry-run", "Architecture and policy analysis"],
"attachments": ["verdicts.json"],
}
assert run.task.context["draft"] is None
async def test_a_draft_sent_with_questions_comes_back_with_the_answer(tmp_path):
draft = sample_submission(K.REQUIREMENTS, note="Drafted for approval.").args
def confirm_then_submit_what_came_back(task):
returned = task.context["draft"]
return [Call("submit_deliverable", returned)]
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[ask_user()],
[delegate(K.REQUIREMENTS)],
[ask_user("Anything else?")],
],
specialist(K.REQUIREMENTS): [
[
Call(
"needs_input",
{
"questions": ["Do you approve this draft?"],
"draft": draft["content"],
"attachments": [
{"path": path, "content": content}
for path, content in draft["attachments"].items()
],
},
)
],
confirm_then_submit_what_came_back,
],
}
)
engine = make_engine(tmp_path, runtime)
paused = await engine.start("Ingestion")
assert paused.state["drafts"]["requirements"]["content"] == draft["content"]
after = await engine.resume(paused.execution_id, {"reply": "Approved."})
second_run = [r for r in runtime.runs if r.role == specialist(K.REQUIREMENTS)][1]
assert second_run.task.context["draft"] == {
"content": draft["content"],
"attachments": draft["attachments"],
}
assert second_run.task.context["answers"][-1]["reply"] == "Approved."
assert after.state["deliverables"]["requirements"]["content"] == draft["content"]
assert after.state["drafts"] == {}
# --- cap on parallel Specialists (iteration 4, step 5) ---------------------------------
async def test_parallel_specialists_never_exceed_the_cap(tmp_path):
import asyncio
from app.adk.runtime import RunLimiter
in_flight = 0
peak = 0
class SlowRuntime(FakeAgentRuntime):
async def run(self, role, task, tools):
nonlocal in_flight, peak
if role.name == "orchestrator":
return await super().run(role, task, tools)
in_flight += 1
peak = max(peak, in_flight)
await asyncio.sleep(0.01)
try:
return await super().run(role, task, tools)
finally:
in_flight -= 1
runtime = SlowRuntime(happy_path_turns())
engine = make_engine(tmp_path, runtime, limiter=RunLimiter(1))
result = await engine.start("Ingestion", pre_approved=True)
assert result.status == "completed"
assert peak == 1
async def test_without_a_cap_independent_specialists_overlap(tmp_path):
import asyncio
in_flight = 0
peak = 0
class SlowRuntime(FakeAgentRuntime):
async def run(self, role, task, tools):
nonlocal in_flight, peak
if role.name == "orchestrator":
return await super().run(role, task, tools)
in_flight += 1
peak = max(peak, in_flight)
await asyncio.sleep(0.01)
try:
return await super().run(role, task, tools)
finally:
in_flight -= 1
runtime = SlowRuntime(happy_path_turns())
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
assert peak == 2
# --- Customer cases group Executions ---------------------------------------------------
async def test_an_execution_in_a_customer_case_writes_under_that_case(tmp_path):
recorder = InMemoryRecorder()
engine = make_engine(
tmp_path, FakeAgentRuntime(happy_path_turns()), recorder=recorder
)
result = await engine.start(
"Ingestion", pre_approved=True, customer_case="acme-webhooks"
)
case_dir = tmp_path / "deliverables" / "cases" / "acme-webhooks"
out = case_dir / "executions" / result.execution_id
assert (out / "solution-architecture-guide.md").is_file()
assert result.state["customer_case"] == "acme-webhooks"
manifest = json.loads((out / "execution.json").read_text(encoding="utf-8"))
assert manifest["customer_case"] == "acme-webhooks"
assert manifest["workflow_request"] == "Ingestion"
assert manifest["intent_version"] == 1
assert manifest["solution_guide"] == "solution-architecture-guide.md"
assert manifest["deliverables"]["deployment_guidance"] == {
"document": "deliverables/deployment_guidance.md",
"attachments": [
"implementation.json",
"terraform/main.tf",
"terraform/variables.tf",
],
}
for entry in manifest["deliverables"].values():
assert (out / entry["document"]).is_file()
assert all((out / name).is_file() for name in entry["attachments"])
assert (
recorder.statuses[result.execution_id][-1]["details"]["customer_case"]
== "acme-webhooks"
)
lessons = (
tmp_path
/ "lessons"
/ "candidates"
/ "acme-webhooks"
/ f"{result.execution_id}.json"
)
assert lessons.is_file()
assert recorder.lessons[result.execution_id]["customer_case"] == "acme-webhooks"
async def test_two_executions_of_one_customer_case_sit_side_by_side(tmp_path):
first = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
"Ingestion", pre_approved=True, customer_case="acme-webhooks"
)
second = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
"Ingestion, revised", pre_approved=True, customer_case="acme-webhooks"
)
case = tmp_path / "deliverables" / "cases" / "acme-webhooks"
assert {path.name for path in (case / "executions").iterdir()} == {
first.execution_id,
second.execution_id,
}
history = json.loads((case / "intent" / "history.json").read_text(encoding="utf-8"))
assert history["customer_case"] == "acme-webhooks"
assert [
(i["version"], i["workflow_request"], i["status"], i["execution"])
for i in history["intents"]
] == [
(1, "Ingestion", "completed", f"executions/{first.execution_id}"),
(2, "Ingestion, revised", "completed", f"executions/{second.execution_id}"),
]
assert "Ingestion, revised" in (case / "intent" / "002.md").read_text()
assert first.state["intent_version"] == 1 and second.state["intent_version"] == 2
# A Customer case groups Executions; it never feeds one into the next.
assert second.state["conversation"][0]["text"] == "Ingestion, revised"
assert "deliverables" in second.state and first.state != second.state
@pytest.mark.parametrize(
"bad", ["../etc", "Acme", "acme/webhooks", "", "-acme", "a" * 65, 7]
)
async def test_a_customer_case_that_is_not_a_safe_slug_is_refused(tmp_path, bad):
engine = make_engine(tmp_path, FakeAgentRuntime(happy_path_turns()))
with pytest.raises(ValueError, match="customer_case"):
await engine.start("Ingestion", pre_approved=True, customer_case=bad)
assert not (tmp_path / "deliverables").exists()
async def test_an_execution_without_a_customer_case_is_grouped_as_unassigned(tmp_path):
result = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
"Ingestion", pre_approved=True
)
assert result.state["customer_case"] == "unassigned"
assert [path.name for path in (tmp_path / "deliverables").iterdir()] == ["cases"]
assert [path.name for path in (tmp_path / "deliverables" / "cases").iterdir()] == [
"unassigned"
]
async def test_the_intent_is_on_record_as_soon_as_an_execution_starts(tmp_path):
runtime = FakeAgentRuntime(
{"orchestrator": [[ask_user("Which region should this run in?")]]}
)
engine = make_engine(tmp_path, runtime)
paused = await engine.start("Ingest signed webhooks", customer_case="acme-webhooks")
assert paused.status == "awaiting_user"
intent = tmp_path / "deliverables" / "cases" / "acme-webhooks" / "intent"
statement = (intent / "001.md").read_text(encoding="utf-8")
assert "Ingest signed webhooks" in statement
assert paused.execution_id in statement
[entry] = json.loads((intent / "history.json").read_text())["intents"]
assert entry["status"] == "in_progress"
assert entry["statement"] == "intent/001.md"
assert entry["execution_id"] == paused.execution_id

View File

@@ -36,12 +36,18 @@ async def test_submission_takes_attachments_as_a_list():
content=GUIDANCE,
attachments=[
{"path": "terraform/main.tf", "content": "resource {}"},
{"path": "terraform/variables.tf", "content": ""},
{"path": "terraform/variables.tf", "content": "variable {}"},
{"path": "implementation.json", "content": "[]"},
],
)
assert result.outcome == Submitted(
GUIDANCE, {"terraform/main.tf": "resource {}", "terraform/variables.tf": ""}
GUIDANCE,
{
"terraform/main.tf": "resource {}",
"terraform/variables.tf": "variable {}",
"implementation.json": "[]",
},
)

View File

@@ -33,14 +33,11 @@ def through_phase_one(ledger: Ledger) -> None:
def through_phase_two(ledger: Ledger) -> None:
through_phase_one(ledger)
produce(
ledger,
K.PRODUCT_SELECTION,
K.ARCHITECTURE_DIAGRAM,
K.ARCHITECTURE_DESCRIPTION,
K.DESIGN_RECOMMENDATIONS,
K.DEPLOYMENT_GUIDANCE,
ledger, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE
)
approve(ledger, P.SOLUTION_ARCHITECTURE)
produce(ledger, K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)
approve(ledger, P.SOLUTION_IMPLEMENTATION)
def through_phase_three(ledger: Ledger) -> None:
@@ -80,18 +77,18 @@ def test_cross_phase_dependency_needs_the_earlier_phase_approved():
def test_independent_work_becomes_ready_together():
ledger = Ledger()
through_phase_one(ledger)
produce(ledger, K.PRODUCT_SELECTION)
produce(
ledger, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE
)
assert set(ledger.ready_work()) == {
K.ARCHITECTURE_DIAGRAM,
K.DESIGN_RECOMMENDATIONS,
}
# Nothing in implementation is ready until the design is approved.
assert ledger.ready_work() == ()
produce(ledger, K.ARCHITECTURE_DIAGRAM)
approve(ledger, P.SOLUTION_ARCHITECTURE)
assert set(ledger.ready_work()) == {
K.ARCHITECTURE_DESCRIPTION,
K.DESIGN_RECOMMENDATIONS,
K.DEPLOYMENT_GUIDANCE,
}
@@ -259,10 +256,12 @@ def test_reopen_cascades_to_dependents_and_withdraws_affected_approvals():
assert ledger.status(K.VALIDATION_RESULTS) is DeliverableStatus.PENDING
assert ledger.status(K.PRODUCT_SELECTION) is DeliverableStatus.SUBMITTED
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
assert ledger.phase_status(P.SOLUTION_ARCHITECTURE) is PhaseStatus.OPEN
# Revising the code leaves the approved design standing.
assert ledger.phase_status(P.SOLUTION_ARCHITECTURE) is PhaseStatus.APPROVED
assert ledger.phase_status(P.SOLUTION_IMPLEMENTATION) is PhaseStatus.OPEN
assert ledger.phase_status(P.SOLUTION_VALIDATION) is PhaseStatus.OPEN
assert set(result.withdrawn_approvals) == {
P.SOLUTION_ARCHITECTURE,
P.SOLUTION_IMPLEMENTATION,
P.SOLUTION_VALIDATION,
}
assert ledger.revisions(K.DEPLOYMENT_GUIDANCE) == 1
@@ -278,15 +277,15 @@ def test_reopen_invalidates_attempts_of_dependents_already_in_progress():
ledger = Ledger()
through_phase_one(ledger)
produce(ledger, K.PRODUCT_SELECTION)
attempt = ledger.start([K.ARCHITECTURE_DIAGRAM], actor="orchestrator")[
K.ARCHITECTURE_DIAGRAM
attempt = ledger.start([K.DESIGN_RECOMMENDATIONS], actor="orchestrator")[
K.DESIGN_RECOMMENDATIONS
]
ledger.reopen(K.PRODUCT_SELECTION, "user prefers GKE", actor="orchestrator")
assert ledger.status(K.ARCHITECTURE_DIAGRAM) is DeliverableStatus.PENDING
assert ledger.status(K.DESIGN_RECOMMENDATIONS) is DeliverableStatus.PENDING
with pytest.raises(LedgerRefusal, match="attempt"):
ledger.submit(K.ARCHITECTURE_DIAGRAM, attempt, actor="diagrammer")
ledger.submit(K.DESIGN_RECOMMENDATIONS, attempt, actor="design-advisor")
def test_reopen_refuses_deliverables_that_were_never_submitted():
@@ -356,3 +355,27 @@ def test_round_trips_through_plain_data():
assert restored.ready_work() == ledger.ready_work()
assert restored.max_revisions == 5
assert restored.events == ledger.events
def test_revising_the_target_architecture_returns_the_code_to_draft():
ledger = Ledger()
through_phase_three(ledger)
result = ledger.reopen(
K.TARGET_ARCHITECTURE, "NFR-1 has no Check on retention", actor="orchestrator"
)
assert set(result.reset) == {
K.TARGET_ARCHITECTURE,
K.ARCHITECTURE_DESCRIPTION,
K.DEPLOYMENT_GUIDANCE,
K.VALIDATION_RESULTS,
}
assert set(result.withdrawn_approvals) == {
P.SOLUTION_ARCHITECTURE,
P.SOLUTION_IMPLEMENTATION,
P.SOLUTION_VALIDATION,
}
# The code cannot be rewritten until the revised design is approved again.
produce(ledger, K.TARGET_ARCHITECTURE)
assert ledger.ready_work() == ()

View File

@@ -0,0 +1,239 @@
"""Lesson candidates: what an Execution keeps about what went wrong in it."""
import json
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
from app.states.lessons import refusal
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
K = DeliverableKind
P = PhaseId
ROLES = {
"orchestrator": RoleSpec(
"orchestrator", "Coordinate.", budget=40, skills=("solution-architecture",)
),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
FAILED = {"check": "K-2", "evidence": "The identity is shared with another service."}
ADDITION = {
"statement": "The service sets a request timeout.",
"applies_to": "E-1",
"reason": "NFR-1 has no Check on timeouts",
}
def delegate(*kinds: DeliverableKind) -> Call:
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
def approve(phase: PhaseId) -> Call:
return Call("request_approval", {"phase": phase.value, "summary": "done"})
def specialist(kind: DeliverableKind) -> str:
return DEPENDENCY_GRAPH[kind].specialist
def failing_validation() -> Call:
call = sample_submission(K.VALIDATION_RESULTS)
verdicts = json.loads(call.args["attachments"]["verdicts.json"])
verdicts["checks"][1] = {**FAILED, "verdict": "fail"}
verdicts["proposed_additions"] = [ADDITION]
return Call(
call.tool,
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
)
def turns_with_one_revision() -> dict:
once = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
return {
**once,
"orchestrator": [
# The first call is refused: requirements are not done yet.
[delegate(K.TECHNICAL_DECOMPOSITION), delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
[
approve(P.SOLUTION_ARCHITECTURE),
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
],
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[
Call(
"reopen",
{"kind": "deployment_guidance", "reason": "Fix the ingress block."},
),
delegate(K.DEPLOYMENT_GUIDANCE),
],
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[approve(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
[approve(P.PACKAGING), Call("finish")],
],
specialist(K.DEPLOYMENT_GUIDANCE): [
[
Call("submit_deliverable", {"content": "No headings yet."}),
sample_submission(K.DEPLOYMENT_GUIDANCE),
],
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
],
specialist(K.VALIDATION_RESULTS): [
[failing_validation()],
[sample_submission(K.VALIDATION_RESULTS)],
],
}
async def run(tmp_path, turns, **options):
recorder = InMemoryRecorder()
engine = ExecutionEngine(
runtime=FakeAgentRuntime(turns),
roles=ROLES,
recorder=recorder,
checkpointer=InMemorySaver(),
output_root=tmp_path,
skill_versions={"solution-architecture": "abc123"},
**options,
)
result = await engine.start("Ingestion", pre_approved=True)
return result, recorder
def of_type(record: dict, kind: str) -> list[dict]:
return [c for c in record["candidates"] if c["type"] == kind]
async def test_a_finished_execution_records_what_went_wrong(tmp_path):
result, recorder = await run(tmp_path, turns_with_one_revision())
assert result.status == "completed"
record = recorder.lessons[result.execution_id]
assert of_type(record, "revision") == [
{
"type": "revision",
"deliverable": "deployment_guidance",
"reason": "Fix the ingress block.",
"requested_by": "orchestrator",
}
]
assert of_type(record, "failed_check") == [
{"type": "failed_check", "attempt": 1, **FAILED}
]
assert of_type(record, "proposed_addition") == [
{"type": "proposed_addition", "attempt": 1, **ADDITION}
]
refusals = of_type(record, "refusal")
assert {(r["role"], r["tool"], r["deliverable"]) for r in refusals} == {
("orchestrator", "delegate", None),
(
specialist(K.DEPLOYMENT_GUIDANCE),
"submit_deliverable",
"deployment_guidance",
),
}
assert all(r["message"].startswith(("Refused", "Not submitted")) for r in refusals)
async def test_the_record_names_the_execution_and_the_skills_loaded(tmp_path):
result, recorder = await run(tmp_path, turns_with_one_revision())
record = recorder.lessons[result.execution_id]
assert record["execution_id"] == result.execution_id
assert record["status"] == "completed"
assert record["role_skills"]["orchestrator"] == ["solution-architecture"]
assert record["skill_versions"] == {"solution-architecture": "abc123"}
async def test_the_record_is_also_written_beside_the_deliverables_tree(tmp_path):
result, recorder = await run(tmp_path, turns_with_one_revision())
path = (
tmp_path
/ "lessons"
/ "candidates"
/ "unassigned"
/ f"{result.execution_id}.json"
)
assert (
json.loads(path.read_text(encoding="utf-8"))
== (recorder.lessons[result.execution_id])
)
async def test_an_execution_that_ends_with_warnings_still_records_lessons(tmp_path):
turns = {
"orchestrator": [[delegate(K.REQUIREMENTS)], []],
specialist(K.REQUIREMENTS): [[]], # stops without submitting
}
result, recorder = await run(tmp_path, turns)
assert result.status == "completed_with_warnings"
record = recorder.lessons[result.execution_id]
assert of_type(record, "unfinished") == [
{
"type": "unfinished",
"deliverable": "requirements",
"role": specialist(K.REQUIREMENTS),
"reason": "stopped",
}
]
assert not (
tmp_path / "deliverables" / "cases" / "unassigned" / "executions"
).exists()
async def test_a_cloud_read_refusal_keeps_only_its_leading_words(tmp_path):
async def list_instances() -> ToolResult:
return ToolResult("Discovery gap: project acme-prod-1234 API not enabled.")
tool = ToolSpec(
"compute__list_instances",
"List instances.",
{"type": "object", "properties": {}},
list_instances,
tier="cloud_read",
)
def factory(role, deliverable, working):
return [tool] if deliverable == "requirements" else []
turns = {
"orchestrator": [[delegate(K.REQUIREMENTS)], []],
specialist(K.REQUIREMENTS): [
[Call("compute__list_instances"), sample_submission(K.REQUIREMENTS)]
],
}
result, recorder = await run(tmp_path, turns, tool_factory=factory, max_revisions=3)
await_record = recorder.lessons[result.execution_id]
# allow_cloud_read is False in a pre-approved Execution, so the gate refuses.
[gap] = of_type(await_record, "refusal")
assert gap["tool"] == "compute__list_instances"
assert "acme-prod-1234" not in gap["message"]
def test_refusal_messages_are_bounded():
item = refusal(
"validator", "validation_results", "terraform_validate", "general", "x" * 900
)
assert len(item["message"]) == 500

View File

@@ -122,15 +122,15 @@ async def test_tool_factory_assembles_local_mcp_and_consultation_tools():
diagrammer = RoleSpec("diagrammer", "", budget=5, tools=("mermaid_check",))
analyst = RoleSpec("requirements-analyst", "", budget=5, mcp_servers=("compute",))
assert [t.name for t in factory(diagrammer, "architecture_diagram")] == [
assert [t.name for t in factory(diagrammer, "architecture_diagram", {})] == [
"mermaid_check"
]
assert [t.name for t in factory(analyst, "requirements")] == [
assert [t.name for t in factory(analyst, "requirements", {})] == [
"compute__list_instances"
]
[consult_tool] = factory(
RoleSpec("design-advisor", "", budget=5), "design_recommendations"
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
)
await consult_tool.handler(brief="b")
assert runtime.runs[0].tools == ("submit_advice", "docs__search")
@@ -140,8 +140,171 @@ async def test_tool_factory_gives_the_consultation_only_to_the_design_advisor():
roles = {PILLAR_ADVISOR: ADVISOR}
factory = make_tool_factory(FakeAgentRuntime({}), roles, PILLARS)
design = factory(RoleSpec("design-advisor", "", budget=5), "design_recommendations")
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition")
design = factory(
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
)
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition", {})
assert [tool.name for tool in design] == [CONSULT_TOOL]
assert list(other) == []
# --- advice remembered across Revisions (iteration 4, step 3) --------------------------
async def test_naming_pillars_reconsults_only_those_and_returns_the_rest_from_memory():
memory: dict = {}
first = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
await consult(
pillar_consultation_tool(first, ADVISOR, PILLARS, memory=memory), brief="v1"
)
def revised(task):
pillar = task.context["pillar"]
return [Call("submit_advice", {"content": f"Revised advice for {pillar}."})]
second = FakeAgentRuntime({PILLAR_ADVISOR: [revised]})
tool = pillar_consultation_tool(second, ADVISOR, PILLARS, memory=memory)
result = await consult(tool, brief="v2", pillars=["reliability"])
assert [run.task.context["pillar"] for run in second.runs] == ["reliability"]
assert "## reliability\n\nRevised advice for reliability." in result.message
assert (
"## security (from your earlier consultation)\n\n"
"Use strong defaults for security." in result.message
)
assert memory["pillar_advice"]["reliability"] == "Revised advice for reliability."
assert set(memory["pillar_advice"]) == set(PILLARS)
async def test_omitting_pillars_consults_every_pillar_afresh():
memory = {"pillar_advice": {pillar: "old" for pillar in PILLARS}}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
result = await consult(tool, brief="v2")
assert len(runtime.runs) == 3
assert "earlier consultation" not in result.message
assert "old" not in memory["pillar_advice"].values()
async def test_failed_advice_is_not_remembered():
memory: dict = {}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [advice("security"), []]})
tool = pillar_consultation_tool(
runtime, ADVISOR, {"security": "s", "reliability": "r"}, memory=memory
)
result = await consult(tool, brief="b")
assert "## reliability\n\nNo guidance" in result.message
assert set(memory["pillar_advice"]) == {"security"}
async def test_the_fan_out_shares_the_cap_and_a_cap_of_one_does_not_deadlock():
import asyncio
from app.adk.runtime import RunLimiter
limiter = RunLimiter(1)
in_flight = 0
peak = 0
class SlowRuntime(FakeAgentRuntime):
async def run(self, role, task, tools):
nonlocal in_flight, peak
in_flight += 1
peak = max(peak, in_flight)
await asyncio.sleep(0.01)
try:
return await super().run(role, task, tools)
finally:
in_flight -= 1
runtime = SlowRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, limiter=limiter)
async with limiter.slot(): # the design advisor's own run holds the only slot
result = await asyncio.wait_for(consult(tool, brief="b"), timeout=5)
assert peak == 1
assert result.message.count("Use strong defaults") == 3
async def test_pillar_advice_survives_a_revision_through_the_engine(tmp_path):
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import sample_submission
from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
K, P = DeliverableKind, PhaseId
design = DEPENDENCY_GRAPH[K.DESIGN_RECOMMENDATIONS].specialist
roles = {
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
PILLAR_ADVISOR: ADVISOR,
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def delegate(*kinds):
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
def approve(phase):
return Call("request_approval", {"phase": phase.value, "summary": "done"})
submit = sample_submission(K.DESIGN_RECOMMENDATIONS)
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[
Call(
"reopen",
{"kind": "design_recommendations", "reason": "Tighten IAM."},
),
delegate(K.DESIGN_RECOMMENDATIONS),
],
[], # stops here: a pre-approved Execution ends with warnings
],
DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist: [
[sample_submission(K.REQUIREMENTS)]
],
DEPENDENCY_GRAPH[K.TECHNICAL_DECOMPOSITION].specialist: [
[sample_submission(K.TECHNICAL_DECOMPOSITION)]
],
DEPENDENCY_GRAPH[K.PRODUCT_SELECTION].specialist: [
[sample_submission(K.PRODUCT_SELECTION)]
],
design: [
[Call(CONSULT_TOOL, {"brief": "v1"}), submit],
[Call(CONSULT_TOOL, {"brief": "v2", "pillars": ["security"]}), submit],
],
PILLAR_ADVISOR: [turn_for_pillar] * 4,
}
)
engine = ExecutionEngine(
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
tool_factory=make_tool_factory(runtime, roles, PILLARS),
)
result = await engine.start("Ingestion", pre_approved=True)
advisor_runs = [r for r in runtime.runs if r.role == PILLAR_ADVISOR]
assert [r.task.context["pillar"] for r in advisor_runs[3:]] == ["security"]
second = [r for r in runtime.runs if r.role == design][1]
assert "## reliability (from your earlier consultation)" in second.results[0][1]
stored = result.state["working"]["design_recommendations"]["pillar_advice"]
assert set(stored) == set(PILLARS)

View File

@@ -12,7 +12,7 @@ def test_approval_prompt_explains_the_structured_reply():
"approval": {
"phase": "solution_architecture",
"summary": "Cloud Run and Pub/Sub",
"deliverables": ["product_selection", "architecture_diagram"],
"deliverables": ["product_selection", "target_architecture"],
}
}
)
@@ -74,3 +74,27 @@ def test_generate_response_maps_deliverables_onto_legacy_artifacts():
assert body["artifacts"]["terraform_code"] == "resource {}"
assert body["artifacts"]["solution_guide"] is None
assert body["warnings"] == ["revision limit"]
def test_the_design_approval_says_what_the_user_is_committing_to():
text = describe_prompt(
{
"approval": {
"phase": "solution_architecture",
"summary": "Cloud Run and Pub/Sub",
"deliverables": ["target_architecture"],
"target_architecture": {
"elements": 9,
"relationships": 11,
"checks": 23,
"not_applicable": [
{"recommendation": "DR-9", "reason": "Single region."}
],
},
}
}
)
assert "9 Elements, 11 Relationships and 23 Checks" in text
assert "No infrastructure code is written until you approve it" in text
assert "Recommendation DR-9 is treated as not applicable: Single region." in text

View File

@@ -0,0 +1,616 @@
"""The Requirements chain: the Target architecture graph and the data files around it.
ADR-0005: the outcome is understood before code is written, and validation answers
the graph's Checks and nothing else.
"""
import json
import pytest
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import SAMPLE_DATA, Call, FakeAgentRuntime, sample_submission
from app.adk.runtime import RoleSpec
from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
from app.tools.execution import specialist_tools
from app.tools.local import mermaid_check
from app.workflows.chain import validation_outcome
from app.workflows.deliverables import (
DEPENDENCY_GRAPH,
DeliverableKind,
PhaseId,
check_submission,
)
from app.workflows.target_architecture import (
GraphError,
chain_gaps,
parse_graph,
render_mermaid,
)
K = DeliverableKind
P = PhaseId
REQUIREMENTS = json.loads(SAMPLE_DATA["requirements.json"])
COMPONENTS = json.loads(SAMPLE_DATA["components.json"])
PRODUCTS = json.loads(SAMPLE_DATA["products.json"])
RECOMMENDATIONS = json.loads(SAMPLE_DATA["recommendations.json"])
def graph_data() -> dict:
return json.loads(SAMPLE_DATA["target-architecture.json"])
def problems(data: dict) -> tuple[str, ...]:
with pytest.raises(GraphError) as raised:
parse_graph(json.dumps(data))
return raised.value.problems
def gaps(data: dict, **overrides) -> tuple[str, ...]:
links = {
"requirements": REQUIREMENTS,
"components": COMPONENTS,
"products": PRODUCTS,
"recommendations": RECOMMENDATIONS,
**overrides,
}
return chain_gaps(parse_graph(json.dumps(data)), **links)
def upstream(*kinds: DeliverableKind, **replaced: str) -> dict:
"""The Deliverables a submission builds on, from the coherent sample."""
built = {}
for kind in kinds:
args = sample_submission(kind).args
attachments = {
name: replaced.get(name, text) for name, text in args["attachments"].items()
}
built[kind.value] = {"content": args["content"], "attachments": attachments}
return built
DESIGN = (
K.REQUIREMENTS,
K.TECHNICAL_DECOMPOSITION,
K.PRODUCT_SELECTION,
K.DESIGN_RECOMMENDATIONS,
)
# --- the graph is well-formed ----------------------------------------------------------
def test_the_sample_graph_parses_into_elements_relationships_and_checks():
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
assert [element.id for element in graph.elements] == ["E-1", "E-2"]
assert graph.elements[1].component is None # a supporting Element
assert [(r.source, r.target) for r in graph.relationships] == [("E-1", "E-2")]
assert graph.check_ids == ("K-1", "K-2")
assert graph.summary() == {
"elements": 2,
"relationships": 1,
"checks": 2,
"not_applicable": [],
}
def test_a_relationship_must_join_two_elements():
data = graph_data()
data["relationships"][0]["to"] = "E-9"
assert problems(data) == (
"target-architecture.json: Relationship R-1 to E-9, which is not an Element",
)
def test_a_check_must_apply_to_something_in_the_graph():
data = graph_data()
data["checks"][0]["applies_to"] = "E-9"
assert problems(data) == (
(
"target-architecture.json: Check K-1 applies to E-9, "
"which is not an Element or Relationship"
),
"target-architecture.json: Element E-1 has no Check",
)
def test_every_element_needs_a_check_and_every_id_is_used_once():
data = graph_data()
data["checks"] = [data["checks"][0]]
data["relationships"][0]["id"] = "E-1"
assert problems(data) == (
"target-architecture.json: id E-1 is used more than once",
"target-architecture.json: Element E-2 has no Check",
)
def test_a_check_without_a_statement_is_named():
data = graph_data()
data["checks"][1]["statement"] = " "
assert (
"target-architecture.json.checks[1]: needs a non-empty statement"
in problems(data)
)
@pytest.mark.parametrize("text", ["not json", "[]", '{"elements": "many"}'])
def test_a_graph_that_is_not_the_right_shape_is_refused(text):
with pytest.raises(GraphError):
parse_graph(text)
# --- the Requirements chain has no gap -------------------------------------------------
def test_the_sample_chain_is_complete():
assert gaps(graph_data()) == ()
def test_a_non_functional_requirement_with_no_check_is_a_gap():
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
assert gaps(graph_data(), requirements=[*REQUIREMENTS, retention]) == (
"Requirement NFR-2 has no Check",
)
def test_a_functional_requirement_is_carried_by_its_components_element():
# FR-1 is the source of no Check, but C-1 serves it and E-1 realises C-1.
assert not [gap for gap in gaps(graph_data()) if "FR-1" in gap]
unserved = [{**COMPONENTS[0], "requirements": ["NFR-1"]}]
assert gaps(graph_data(), components=unserved) == ("Requirement FR-1 has no Check",)
def test_a_component_with_no_element_is_a_gap():
replay = {"id": "C-2", "name": "Replay service", "requirements": []}
assert gaps(graph_data(), components=[*COMPONENTS, replay]) == (
"Component C-2 has no Element",
)
def test_a_selected_product_that_no_element_uses_is_a_gap():
armour = [{"component": "C-1", "product": "Example Armor"}]
assert gaps(graph_data(), products=armour) == (
"Product Example Armor selected for C-1 appears on no Element",
)
def test_a_recommendation_needs_a_check_or_a_reason_it_does_not_apply():
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
recommendations = [*RECOMMENDATIONS, cmek]
assert gaps(graph_data(), recommendations=recommendations) == (
"Recommendation DR-2 has no Check",
)
excused = graph_data()
excused["not_applicable"] = [
{"recommendation": "DR-2", "reason": "No regulated data."}
]
assert gaps(excused, recommendations=recommendations) == ()
assert parse_graph(json.dumps(excused)).summary()["not_applicable"] == [
{"recommendation": "DR-2", "reason": "No regulated data."}
]
def test_a_check_must_come_from_a_requirement_or_recommendation():
data = graph_data()
data["checks"][0]["source"] = "my own idea"
assert gaps(data) == (
(
"Check K-1 has source my own idea, which is not a requirement or design "
"recommendation"
),
"Requirement NFR-1 has no Check",
)
def test_a_target_architecture_with_a_gap_cannot_be_submitted():
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
built_on = upstream(
*DESIGN, **{"requirements.json": json.dumps([*REQUIREMENTS, retention])}
)
args = sample_submission(K.TARGET_ARCHITECTURE).args
assert check_submission(
K.TARGET_ARCHITECTURE, args["content"], args["attachments"], built_on
) == ("Requirement NFR-2 has no Check",)
assert (
check_submission(
K.TARGET_ARCHITECTURE,
args["content"],
args["attachments"],
upstream(*DESIGN),
)
== ()
)
# --- the links before the graph --------------------------------------------------------
def test_requirements_need_ids_and_a_kind():
def missing(items) -> tuple[str, ...]:
args = sample_submission(K.REQUIREMENTS).args
return check_submission(
K.REQUIREMENTS, args["content"], {"requirements.json": json.dumps(items)}
)
assert missing(REQUIREMENTS) == ()
assert missing([]) == ("requirements.json: lists no requirements",)
assert missing([REQUIREMENTS[0], {**REQUIREMENTS[0], "kind": "must"}]) == (
"requirements.json: id FR-1 is used more than once",
"requirements.json: FR-1 kind must be one of functional, non_functional",
)
assert missing([{"id": "FR-1", "kind": "functional"}]) == (
"requirements.json[0]: needs text",
)
def test_every_functional_requirement_is_served_by_a_component():
args = sample_submission(K.TECHNICAL_DECOMPOSITION).args
def missing(components) -> tuple[str, ...]:
return check_submission(
K.TECHNICAL_DECOMPOSITION,
args["content"],
{"components.json": json.dumps(components)},
upstream(K.REQUIREMENTS),
)
assert missing(COMPONENTS) == ()
assert missing([{"id": "C-1", "name": "Ingress", "requirements": ["NFR-9"]}]) == (
"components.json: C-1 serves NFR-9, which is not a requirement",
"components.json: functional requirement FR-1 is served by no component",
)
def test_every_component_gets_a_product_and_recommendations_name_real_components():
selection = sample_submission(K.PRODUCT_SELECTION).args
advice = sample_submission(K.DESIGN_RECOMMENDATIONS).args
built_on = upstream(K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
assert check_submission(
K.PRODUCT_SELECTION,
selection["content"],
{"products.json": json.dumps([{"component": "C-7", "product": "X"}])},
built_on,
) == (
"products.json: C-7 is not a component",
"products.json: component C-1 has no product selected",
)
wandering = [{**RECOMMENDATIONS[0], "components": ["C-7"]}]
assert check_submission(
K.DESIGN_RECOMMENDATIONS,
advice["content"],
{"recommendations.json": json.dumps(wandering)},
built_on,
) == ("recommendations.json: DR-1 names C-7, which is not a component",)
# --- code and validation answer the graph ----------------------------------------------
EVERYTHING_BEFORE_CODE = (*DESIGN, K.TARGET_ARCHITECTURE)
def test_deployment_guidance_must_account_for_every_check():
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
partial = json.dumps(
[
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
{"check": "K-7", "file": "terraform/elsewhere.tf", "address": "b"},
]
)
missing = check_submission(
K.DEPLOYMENT_GUIDANCE,
args["content"],
{**args["attachments"], "implementation.json": partial},
upstream(*EVERYTHING_BEFORE_CODE),
)
assert missing == (
"implementation.json: K-7 is not a Check in the Target architecture",
"implementation.json: Check K-2 is not accounted for",
"implementation.json: K-7 names terraform/elsewhere.tf, which is not attached",
)
def verdicts(**changes) -> str:
data = json.loads(SAMPLE_DATA["verdicts.json"])
data.update(changes)
return json.dumps(data)
def validation_missing(text: str) -> tuple[str, ...]:
args = sample_submission(K.VALIDATION_RESULTS).args
return check_submission(
K.VALIDATION_RESULTS,
args["content"],
{"verdicts.json": text},
upstream(*EVERYTHING_BEFORE_CODE, K.DEPLOYMENT_GUIDANCE),
)
def test_validation_gives_exactly_one_verdict_per_check():
one = {"check": "K-1", "verdict": "pass", "evidence": "seen"}
invented = {"check": "K-9", "verdict": "fail", "evidence": "No Cloud Armor rule."}
assert validation_missing(SAMPLE_DATA["verdicts.json"]) == ()
assert validation_missing(verdicts(checks=[one, one, invented])) == (
"verdicts.json.checks: check K-1 is used more than once",
"verdicts.json.checks: K-9 is not a Check in the Target architecture",
"verdicts.json.checks: Check K-2 is not accounted for",
)
def test_the_validator_cannot_fail_the_code_on_something_the_graph_lacks():
"""What it notices beyond the Checks is a proposed addition, never a failure."""
addition = {
"statement": "Cloud Armor rate limiting is configured.",
"applies_to": "E-1",
"reason": "NFR-1",
}
text = verdicts(proposed_additions=[addition])
assert validation_missing(text) == ()
outcome = validation_outcome(text)
assert outcome["failed"] == []
assert outcome["proposed_additions"] == [addition]
def test_a_verdict_needs_evidence_and_a_known_value_and_the_dry_run_a_result():
vague = [
{"check": "K-1", "verdict": "probably", "evidence": "x"},
{"check": "K-2", "verdict": "pass", "evidence": " "},
]
assert validation_missing(verdicts(dry_run={"verdict": "ok"}, checks=vague)) == (
"verdicts.json.dry_run: needs a verdict (pass, fail, not_run) and evidence",
"verdicts.json.checks[1]: needs evidence",
"verdicts.json.checks: K-1 verdict must be one of pass, fail, not_verifiable",
"verdicts.json.checks: Check K-2 is not accounted for",
)
def test_a_validation_outcome_separates_faults_in_code_from_gaps_in_the_graph():
checks = [
{"check": "K-1", "verdict": "fail", "evidence": "min instances is 1"},
{"check": "K-2", "verdict": "not_verifiable", "evidence": "No such argument."},
]
outcome = validation_outcome(verdicts(checks=checks))
assert outcome["failed"] == [{"check": "K-1", "evidence": "min instances is 1"}]
assert outcome["not_verifiable"] == [
{"check": "K-2", "evidence": "No such argument."}
]
assert outcome["passed"] == []
assert outcome["dry_run"]["verdict"] == "pass"
# --- the diagram is a view of the graph ------------------------------------------------
async def test_the_diagram_shows_every_element_and_relationship_once():
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
diagram = render_mermaid(graph)
assert diagram == render_mermaid(graph) # deterministic
assert diagram.count("Ingress service") == 1
assert diagram.count("Ingress identity") == 1
assert diagram.count("-->") == 1
assert "runs as" in diagram
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
async def test_names_with_characters_mermaid_reserves_still_render():
data = graph_data()
data["elements"][0]["name"] = 'Ingest "edge" [v2] (public) | {x}'
data["elements"][0]["id"] = "svc/ingest.v2"
data["relationships"][0]["from"] = "svc/ingest.v2"
data["checks"][0]["applies_to"] = "svc/ingest.v2"
data["relationships"][0]["kind"] = "calls (HTTPS) | signed"
diagram = render_mermaid(parse_graph(json.dumps(data)))
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
assert "svc/ingest.v2" not in diagram
async def test_the_engine_draws_the_diagram_when_the_target_architecture_is_submitted():
submit = specialist_tools(
K.TARGET_ARCHITECTURE,
extra=[],
allow_cloud_read=False,
upstream=upstream(*DESIGN),
)[0]
args = sample_submission(K.TARGET_ARCHITECTURE).args
result = await submit.handler(**args)
assert result.outcome.attachments["architecture.mmd"] == render_mermaid(
parse_graph(SAMPLE_DATA["target-architecture.json"])
)
async def test_a_gap_is_refused_to_the_specialist_with_the_missing_link_named():
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
built_on = upstream(
*DESIGN, **{"recommendations.json": json.dumps([*RECOMMENDATIONS, cmek])}
)
submit = specialist_tools(
K.TARGET_ARCHITECTURE, extra=[], allow_cloud_read=False, upstream=built_on
)[0]
result = await submit.handler(**sample_submission(K.TARGET_ARCHITECTURE).args)
assert result.outcome is None
assert result.message == (
"Not submitted. The Deliverable is missing: Recommendation DR-2 has no Check"
)
# --- through the engine ----------------------------------------------------------------
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def delegate(*kinds: DeliverableKind) -> Call:
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
def approve(phase: PhaseId) -> Call:
return Call("request_approval", {"phase": phase.value, "summary": "done"})
def specialist(kind: DeliverableKind) -> str:
return DEPENDENCY_GRAPH[kind].specialist
def to_design_approval() -> list[list[Call]]:
return [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
]
def engine_for(tmp_path, turns: dict) -> tuple[ExecutionEngine, FakeAgentRuntime]:
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
)
return engine, runtime
async def test_the_user_approves_the_graph_before_any_code_is_written(tmp_path):
once = {
specialist(kind): [[sample_submission(kind)]]
for kind in (*DESIGN, K.TARGET_ARCHITECTURE)
}
engine, runtime = engine_for(
tmp_path,
{
**once,
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY)],
[delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
# Tries to start the code before asking: refused, then asks.
[delegate(K.DEPLOYMENT_GUIDANCE), approve(P.SOLUTION_ARCHITECTURE)],
],
},
)
first = await engine.start("Ingest webhooks")
paused = await engine.resume(
first.execution_id,
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
)
assert paused.prompt["approval"]["phase"] == "solution_architecture"
assert paused.prompt["approval"]["target_architecture"] == {
"elements": 2,
"relationships": 1,
"checks": 2,
"not_applicable": [],
}
refused = runtime.runs[-1].results[0]
assert refused[0] == "delegate" and refused[1].startswith("Refused")
assert specialist(K.DEPLOYMENT_GUIDANCE) not in [run.role for run in runtime.runs]
async def test_the_orchestrator_is_told_what_validation_decided(tmp_path):
failing = json.loads(SAMPLE_DATA["verdicts.json"])
failing["checks"][0] = {
"check": "K-1",
"verdict": "fail",
"evidence": "min instances is 1",
}
failed_validation = sample_submission(K.VALIDATION_RESULTS)
failed_validation = Call(
failed_validation.tool,
{
**failed_validation.args,
"attachments": {"verdicts.json": json.dumps(failing)},
},
)
once = {
specialist(kind): [[sample_submission(kind)]]
for kind in (*DESIGN, K.TARGET_ARCHITECTURE, K.ARCHITECTURE_DESCRIPTION)
}
engine, runtime = engine_for(
tmp_path,
{
**once,
"orchestrator": [
*to_design_approval(),
[
approve(P.SOLUTION_ARCHITECTURE),
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
],
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[
Call(
"reopen",
{
"kind": "deployment_guidance",
"reason": "K-1 failed: min instances is 1",
},
),
delegate(K.DEPLOYMENT_GUIDANCE),
],
[],
],
specialist(K.DEPLOYMENT_GUIDANCE): [
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
],
specialist(K.VALIDATION_RESULTS): [[failed_validation]],
},
)
result = await engine.start("Ingest webhooks", pre_approved=True)
turns = [run for run in runtime.runs if run.role == "orchestrator"]
before, after = turns[6].task.context, turns[7].task.context
assert before["validation"] is None
assert after["validation"]["failed"] == [
{"check": "K-1", "evidence": "min instances is 1"}
]
assert after["validation"]["proposed_additions"] == []
# Revising the code leaves the approved design standing.
phases = result.state["ledger"]["phases"]
assert phases["solution_architecture"] == "approved"
revised = [
run for run in runtime.runs if run.role == specialist(K.DEPLOYMENT_GUIDANCE)
][1]
assert revised.task.context["feedback"] == ["K-1 failed: min instances is 1"]

View File

@@ -1,59 +0,0 @@
import pathlib
import unittest
ROOT = pathlib.Path(__file__).parents[1]
def get_artifact_text(filename):
search_dirs = [
ROOT / "deliverables" / "as-is",
ROOT / "deliverables" / "target",
ROOT / "deliverables" / "validation",
ROOT / "deliverables" / "guides",
ROOT / "deliverables",
ROOT / "docs",
ROOT,
]
for d in search_dirs:
candidate = d / filename
if candidate.is_file():
return candidate.read_text(encoding="utf-8")
raise FileNotFoundError(f"Artifact {filename} not found")
class ArtifactTests(unittest.TestCase):
def test_requirements_defer_products(self):
text = get_artifact_text("requirements.md")
self.assertIn("Product selection deferred:", text)
self.assertIn("Open questions", text)
def test_architecture_has_products_and_flow(self):
text = get_artifact_text("architecture.md")
for product in ("Cloud Run", "Pub/Sub", "Cloud Storage", "Firestore"):
self.assertIn(product, text)
def test_mermaid_is_flowchart(self):
text = get_artifact_text("architecture.mmd")
self.assertTrue(text.startswith("flowchart"))
self.assertIn("Cloud Run", text)
def test_terraform_root_is_complete(self):
main = (ROOT / "terraform/main.tf").read_text()
self.assertIn("google_cloud_run_v2_service", main)
self.assertIn("google_pubsub_topic", main)
self.assertNotRegex(main, r"(?i)(password|secret|private_key)\\s*=")
def test_guide_packages_outputs(self):
guide = get_artifact_text("solution-architecture-guide.md")
for heading in (
"Requirements",
"Architecture",
"Terraform",
"Validation",
"Verification",
):
self.assertIn(heading, guide)
if __name__ == "__main__":
unittest.main()

View File

@@ -1,20 +1,8 @@
import subprocess
from pathlib import Path
ROOT = Path(__file__).parents[1]
def test_artifact_validator():
result = subprocess.run(
["python3", "scripts/validate_artifacts.py"],
cwd=ROOT,
capture_output=True,
check=False,
text=True,
)
assert result.returncode == 0, result.stdout + result.stderr
def test_no_secret_values():
assert (
"replace-with-project-id"

View File

@@ -1,6 +1,7 @@
"""Local tools: Mermaid check and Terraform fmt/validate/plan in a scratch directory."""
import shutil
from pathlib import Path
import pytest
@@ -157,3 +158,88 @@ def test_local_tools_are_built_by_name_with_their_tier():
}
with pytest.raises(KeyError, match="terraform_apply"):
local_tools(["terraform_apply"], TerraformRunner())
# --- the provider cache ----------------------------------------------------------------
def fake_terraform(tmp_path: Path) -> tuple[Path, Path]:
"""A stand-in terraform that records the cache settings it was run with."""
log = tmp_path / "calls.log"
binary = tmp_path / "terraform"
binary.write_text(
"#!/bin/sh\n"
f'echo "$1 cache=$TF_PLUGIN_CACHE_DIR '
f'reuse=$TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE" >> {log}\n'
'echo "Success! The configuration is valid."\n'
)
binary.chmod(0o755)
return binary, log
async def test_providers_are_cached_in_a_directory_the_runner_creates(tmp_path):
binary, log = fake_terraform(tmp_path)
cache = tmp_path / "not" / "yet" / "there"
runner = TerraformRunner(binary=str(binary), plugin_cache_dir=str(cache))
result = await runner.validate(VALID_TF)
assert result.message.startswith("terraform validate passed")
assert cache.is_dir()
# Scratch directories have no lock file, so reuse must be allowed explicitly.
assert log.read_text().splitlines() == [
f"init cache={cache} reuse=true",
f"validate cache={cache} reuse=true",
]
async def test_without_a_cache_directory_terraform_is_left_to_its_defaults(tmp_path):
binary, log = fake_terraform(tmp_path)
await TerraformRunner(binary=str(binary)).validate(VALID_TF)
assert log.read_text().splitlines() == [
"init cache= reuse=",
"validate cache= reuse=",
]
async def test_concurrent_checks_install_into_the_cache_one_at_a_time(tmp_path):
import asyncio
marker = tmp_path / "installing"
overlaps = tmp_path / "overlaps"
binary = tmp_path / "terraform"
binary.write_text(
"#!/bin/sh\n"
'if [ "$1" = init ]; then\n'
f" [ -e {marker} ] && echo overlap >> {overlaps}\n"
f" touch {marker}; sleep 0.2; rm -f {marker}\n"
"fi\n"
"echo ok\n"
)
binary.chmod(0o755)
runner = TerraformRunner(binary=str(binary), plugin_cache_dir=str(tmp_path / "c"))
await asyncio.gather(*(runner.validate(VALID_TF) for _ in range(3)))
assert not overlaps.exists()
def test_the_cache_setting_resolves_under_the_repository_and_can_be_disabled():
from app.config import Settings
default = Settings(_env_file=None)
assert default.terraform_plugin_cache() == str(
default.BASE_DIR / ".cache" / "terraform-plugins"
)
assert (
Settings(_env_file=None, TF_PLUGIN_CACHE_DIR="").terraform_plugin_cache()
is None
)
assert (
Settings(
_env_file=None, TF_PLUGIN_CACHE_DIR="/var/cache/tf"
).terraform_plugin_cache()
== "/var/cache/tf"
)