feat: working orhcestration engine

This commit is contained in:
2026-10-21 19:50:24 +01:00
commit ce5e2f20cd
161 changed files with 31794 additions and 0 deletions

View File

View File

@@ -0,0 +1,102 @@
"""Every Execution is judged as it ends (iteration 5 step 19)."""
import json
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import FakeAgentRuntime, FakeDecider
from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from tests.reference_workflow.assessment import QUALITY_RUBRIC, assess
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
WORKFLOW,
sample_submission,
)
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
ADVICE = {"design_recommendations": {"content": "Use two zones for the database."}}
def graded(level: float) -> FakeDecider:
return FakeDecider({key: Answer(level, 0.9) for key in QUALITY_RUBRIC})
async def test_the_rubric_is_scored_from_nought_to_one_with_its_mean():
decider = graded(3.0)
decider.answers["specific"] = Answer(1.5, 0.9)
assessment = await assess(decider, ADVICE)
assert assessment == {
"recommendations_quality": {
"specific": 0.5,
"actionable": 1.0,
"trade_offs": 1.0,
"traceable": 1.0,
},
"mean": 0.88,
"meets_bar": True,
}
document, _, _ = decider.asked[0]
assert document == "Use two zones for the database."
async def test_a_weak_design_is_reported_as_below_the_bar():
assessment = await assess(graded(1.0), ADVICE)
assert assessment["mean"] == 0.33 and assessment["meets_bar"] is False
async def test_nothing_to_judge_or_no_decision_gives_no_assessment():
assert await assess(graded(3.0), {}) is None
assert await assess(NoDecider(), ADVICE) is None
def engine(tmp_path, decider, **options) -> ExecutionEngine:
return ExecutionEngine(
workflow=WORKFLOW,
runtime=FakeAgentRuntime(
{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
),
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
decider=decider,
**options,
)
async def test_a_completed_execution_carries_its_assessment(tmp_path):
result = await engine(tmp_path, graded(1.0), assess_at_end=True).start(
"Ingest webhooks", pre_approved=True
)
# A low score is reported and the Execution still completes.
assert result.status == "completed"
assert result.state["assessment"]["meets_bar"] is False
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
assert manifest["assessment"] == result.state["assessment"]
async def test_an_execution_is_not_judged_unless_that_is_switched_on(tmp_path):
decider = graded(3.0)
result = await engine(tmp_path, decider).start("Ingest webhooks", pre_approved=True)
assert result.state["assessment"] is None and decider.asked == []
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
assert manifest["assessment"] is None

View File

@@ -0,0 +1,772 @@
"""Change requests (ADR-0006): an Execution that starts from a Baseline.
A Baseline is a completed Execution of the same Customer case. A Change request is a
new Execution that begins with the Baseline's Deliverables; the Baseline itself is
never modified.
"""
import hashlib
import json
from pathlib import Path
import pytest
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.runtime import NoDecider, RoleSpec
from execution_engine.agent import ExecutionEngine
from execution_engine.baseline import BaselineError, load_baseline
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
SAMPLE_DATA,
WORKFLOW,
DeliverableKind,
sample_submission,
)
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
K = DeliverableKind
CASE = "acme-webhooks"
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def once() -> dict:
return {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
def revised(**changes) -> Call:
"""The requirements of a Change request: the Baseline's, each marked, plus what
`changes` adds or alters by id."""
requirements = json.loads(SAMPLE_DATA["requirements.json"])
for item in requirements:
item["change"] = "unchanged"
if item["id"] in changes:
item.update(changes.pop(item["id"]))
requirements += [{"id": key, **value} for key, value in changes.items()]
call = sample_submission(K.REQUIREMENTS)
return Call(
call.tool,
{
**call.args,
"attachments": {"requirements.json": json.dumps(requirements)},
},
)
RETENTION = {
"NFR-1": {
"change": "changed",
"text": "99.99% availability.",
"reason": "The service is now customer-facing.",
}
}
def changed(**overrides) -> dict:
"""Scripted turns for a Change request that changes NFR-1 and nothing else."""
turns = once()
turns["requirements-analyst"] = [[revised(**json.loads(json.dumps(RETENTION)))]]
turns.update(overrides)
return turns
def engine_for(tmp_path: Path, turns: dict, **options) -> ExecutionEngine:
return ExecutionEngine(
workflow=WORKFLOW,
runtime=FakeAgentRuntime(turns),
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
**options,
)
async def delivered(tmp_path: Path, case: str = CASE, execution_id: str = "v1"):
"""A completed Execution on disk, to be a Baseline."""
result = await engine_for(tmp_path, once()).start(
"Ingest webhooks",
customer_case=case,
pre_approved=True,
execution_id=execution_id,
)
assert result.status == "completed"
return result
def digest(folder: Path) -> dict[str, str]:
return {
str(path.relative_to(folder)): hashlib.sha256(path.read_bytes()).hexdigest()
for path in sorted(folder.rglob("*"))
if path.is_file()
}
def history(tmp_path: Path, case: str = CASE) -> list[dict]:
path = tmp_path / "deliverables" / "cases" / case / "intent" / "history.json"
return json.loads(path.read_text())["intents"]
# --- step 2: start against a Baseline --------------------------------------------------
async def test_a_baseline_is_read_from_its_files(tmp_path):
await delivered(tmp_path)
baseline = load_baseline(tmp_path, CASE, "v1")
assert baseline["execution_id"] == "v1"
assert baseline["workflow_request"] == "Ingest webhooks"
assert set(baseline["deliverables"]) == {kind.value for kind in DEPENDENCY_GRAPH}
graph = baseline["deliverables"]["target_architecture"]["attachments"]
assert "target-architecture.json" in graph
assert baseline["deliverables"]["requirements"]["content"].startswith("#")
async def test_a_change_request_starts_with_its_baseline_and_is_recorded(tmp_path):
await delivered(tmp_path)
result = await engine_for(tmp_path, changed()).start(
"Retention goes from 30 days to 7 years.",
customer_case=CASE,
pre_approved=True,
execution_id="v2",
baseline="v1",
)
assert result.status == "completed"
assert result.state["baseline"]["execution_id"] == "v1"
first, second = history(tmp_path)
assert first["baseline"] is None
assert second["baseline"] == "v1" and second["execution_id"] == "v2"
case = tmp_path / "deliverables" / "cases" / CASE
manifest = json.loads((case / "executions" / "v2" / "execution.json").read_text())
assert manifest["baseline"] == "v1"
assert "- Changes: `v1`" in (case / "intent" / "002.md").read_text()
async def test_the_baseline_is_never_modified(tmp_path):
await delivered(tmp_path)
folder = tmp_path / "deliverables" / "cases" / CASE / "executions" / "v1"
before = digest(folder)
await engine_for(tmp_path, changed()).start(
"Add a second region.",
customer_case=CASE,
pre_approved=True,
execution_id="v2",
baseline="v1",
)
assert digest(folder) == before
async def test_a_change_request_may_itself_be_a_baseline(tmp_path):
await delivered(tmp_path)
for version, previous in (("v2", "v1"), ("v3", "v2")):
await engine_for(tmp_path, changed()).start(
"Another change.",
customer_case=CASE,
pre_approved=True,
execution_id=version,
baseline=previous,
)
assert [item["baseline"] for item in history(tmp_path)] == [None, "v1", "v2"]
assert load_baseline(tmp_path, CASE, "v3")["baseline"] == "v2"
async def test_without_a_baseline_an_execution_starts_clean(tmp_path):
result = await delivered(tmp_path)
assert result.state["baseline"] is None
manifest = json.loads(
(
tmp_path / "deliverables/cases" / CASE / "executions/v1/execution.json"
).read_text()
)
assert manifest["baseline"] is None
@pytest.mark.parametrize(
"case, baseline, message",
[
(CASE, "nope", "is not an Execution of Customer case acme-webhooks"),
("other-client", "v1", "belongs to another Customer case"),
(None, "v1", "needs a customer_case"),
(CASE, "../v1", "is not an Execution id"),
],
)
async def test_a_baseline_that_cannot_be_one_refuses_the_start(
tmp_path, case, baseline, message
):
await delivered(tmp_path)
engine = engine_for(tmp_path, once())
with pytest.raises(BaselineError, match=message):
await engine.start(
"A change.", customer_case=case, execution_id="v2", baseline=baseline
)
# Refused before anything ran or was recorded.
assert [item["execution_id"] for item in history(tmp_path)] == ["v1"]
assert not (tmp_path / "deliverables/cases/other-client").exists()
async def test_an_execution_that_did_not_complete_cannot_be_a_baseline(tmp_path):
from tests.execution.test_router import failing
turns = once()
turns["validator"] = [[failing("K-1")]]
engine = engine_for(tmp_path, turns, max_revisions=0)
unfinished = await engine.start(
"Ingest webhooks", customer_case=CASE, pre_approved=True, execution_id="v1"
)
assert unfinished.status == "completed_with_warnings"
with pytest.raises(BaselineError, match="did not complete"):
load_baseline(tmp_path, CASE, "v1")
# --- step 3: the requirements say what changed -------------------------------------------
def requirements_check(revised_call: Call, baseline=None) -> tuple[str, ...]:
from tests.reference_workflow.workflow import check_submission
baseline = baseline or {
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
}
return check_submission(
K.REQUIREMENTS,
revised_call.args["content"],
revised_call.args["attachments"],
baseline=baseline,
)
def test_a_revised_requirement_list_that_accounts_for_every_change_is_accepted():
call = revised(
**{
"NFR-1": RETENTION["NFR-1"],
"NFR-2": {
"kind": "non_functional",
"text": "A second region.",
"conditions": ["The service runs in two regions."],
"change": "added",
"reason": "Customers are now in two continents.",
},
}
)
assert requirements_check(call) == ()
def test_every_requirement_in_a_change_request_says_whether_it_changed():
call = sample_submission(K.REQUIREMENTS) # no `change` on anything
missing = requirements_check(call)
assert "requirements.json: FR-1 needs a `change`" in missing[0]
assert len(missing) == 2
def test_a_change_needs_its_reason_and_an_unchanged_requirement_its_text():
no_reason = revised(**{"NFR-1": {"change": "changed", "text": "99.99%."}})
quietly = revised(**{"NFR-1": {"text": "99.99% availability."}})
quietly_other = revised(
**{"NFR-1": RETENTION["NFR-1"], "FR-1": {"text": "Accept events and files."}}
)
assert requirements_check(no_reason) == (
"requirements.json: NFR-1 is changed and needs a `reason`",
)
assert (
"NFR-1 is marked unchanged, but its text differs"
in (requirements_check(quietly)[0])
)
assert (
"FR-1 is marked unchanged, but its text differs"
in (requirements_check(quietly_other)[0])
)
def test_a_baseline_requirement_cannot_silently_disappear_or_be_reused():
call = revised(**{"NFR-1": RETENTION["NFR-1"]})
requirements = json.loads(call.args["attachments"]["requirements.json"])
dropped = Call(
call.tool,
{
**call.args,
"attachments": {"requirements.json": json.dumps(requirements[1:])},
},
)
reused = revised(
**{
"NFR-1": {
"change": "added",
"text": "Something else entirely.",
"reason": "New.",
}
}
)
invented = revised(
**{
"NFR-1": RETENTION["NFR-1"],
"NFR-9": {
"kind": "non_functional",
"text": "x",
"conditions": ["x"],
"change": "changed",
"reason": "y",
},
}
)
assert "Baseline requirement FR-1 is missing" in requirements_check(dropped)[0]
assert (
"NFR-1 is marked added, but the Baseline has that id"
in (requirements_check(reused)[0])
)
assert (
"NFR-9 is marked changed, but the Baseline has no such requirement"
in (requirements_check(invented)[0])
)
def test_a_change_request_must_change_something():
assert (
"nothing is marked added, changed or removed"
in (requirements_check(revised())[0])
)
def test_a_removed_requirement_stays_on_record_and_asks_nothing_downstream():
from tests.reference_workflow.chain import read_list
call = revised(
**{"NFR-1": {"change": "removed", "reason": "No longer a requirement."}}
)
upstream = {"requirements": {"attachments": call.args["attachments"]}}
assert requirements_check(call) == ()
live = read_list(upstream, "requirements", "requirements.json")
assert [item["id"] for item in live] == ["FR-1"]
assert '"NFR-1"' in call.args["attachments"]["requirements.json"]
# --- step 4: later Specialists revise from the Baseline ---------------------------------
async def test_each_specialist_is_given_its_delivered_version_and_what_changed(
tmp_path,
):
await delivered(tmp_path)
runtime = FakeAgentRuntime(changed())
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
await engine.start(
"Availability goes up.",
customer_case=CASE,
pre_approved=True,
execution_id="v2",
baseline="v1",
)
by_role = {run.role: run for run in runtime.runs}
analyst = by_role["requirements-analyst"].task.context["baseline"]
assert analyst["execution_id"] == "v1"
assert analyst["original_request"] == "Ingest webhooks"
assert "requirements.json" in analyst["deliverable"]["attachments"]
assert analyst["changes"] is None # nothing has changed yet
architect = by_role["target-architect"].task.context
assert architect["baseline"]["changes"]["requirements"] == [
{
"id": "NFR-1",
"change": "changed",
"reason": "The service is now customer-facing.",
"text": "99.99% availability.",
}
]
# Its working copy already holds the delivered graph.
assert architect["baseline"]["deliverable"]["attachments"][
"target-architecture.json"
]
assert "graph_read" in by_role["target-architect"].tools
# The guide is assembled afresh: the Baseline's is not handed back.
assert by_role["packager"].task.context["baseline"]["deliverable"] is None
async def test_a_description_whose_design_did_not_change_shape_is_carried_over(
tmp_path,
):
await delivered(tmp_path)
runtime = FakeAgentRuntime(changed())
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
result = await engine.start(
"Availability goes up.",
customer_case=CASE,
pre_approved=True,
execution_id="v2",
baseline="v1",
)
# The same Elements and Relationships as the Baseline: the describer is not run.
assert "describer" not in {run.role for run in runtime.runs}
assert result.state["deliverables"]["architecture_description"]["content"]
async def test_a_code_authors_working_copy_starts_as_the_delivered_code(tmp_path):
await delivered(tmp_path)
seen: dict = {}
def author(task):
seen["working_copy"] = task.context["working_copy"]
return [sample_submission(K.DEPLOYMENT_GUIDANCE)]
engine = engine_for(tmp_path, changed(**{"iac-author": [author]}))
await engine.start(
"Availability goes up.",
customer_case=CASE,
pre_approved=True,
execution_id="v2",
baseline="v1",
)
# It is told nothing of a working copy on a first attempt; the tools hold it.
assert seen["working_copy"] is None
# --- what changed, worked out from the ids ------------------------------------------------
def versions(graph_change=None, requirement_change=None) -> tuple[dict, dict]:
"""A Baseline and a revised set of Deliverables, as data files."""
def deliverables() -> dict:
return {
kind: {"attachments": {name: SAMPLE_DATA[name]}}
for kind, name in (
("requirements", "requirements.json"),
("technical_decomposition", "components.json"),
("design_recommendations", "recommendations.json"),
("target_architecture", "target-architecture.json"),
)
}
before, after = deliverables(), deliverables()
change = requirement_change or RETENTION
after["requirements"]["attachments"] = revised(
**json.loads(json.dumps(change))
).args["attachments"]
if graph_change:
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
graph_change(graph)
after["target_architecture"]["attachments"] = {
"target-architecture.json": json.dumps(graph)
}
return before, after
def test_a_change_inside_the_reach_of_a_changed_requirement_is_requested():
from tests.reference_workflow.changes import change_summary
def tighten(graph):
graph["checks"][0]["statement"] = "The service runs at least three instances."
summary = change_summary(*versions(tighten))
assert [item["id"] for item in summary["requirements"]] == ["NFR-1"]
assert summary["checks"] == {"added": [], "changed": ["K-1"], "removed": []}
assert summary["elements"] == {"added": [], "changed": [], "removed": []}
assert summary["unrequested"] == {}
def test_a_change_no_changed_requirement_reaches_is_listed_as_unrequested():
from tests.reference_workflow.changes import change_summary
def wander(graph):
# K-2 comes from DR-1, which did not change, on E-2, which realises nothing
# the changed requirement touches. NFR-1 reaches K-1 and E-1 only.
graph["checks"][1]["statement"] = "The identity has no keys."
change = {
"NFR-1": {"change": "changed", "text": "99.99%.", "reason": "Customer-facing."}
}
before, after = versions(wander, change)
# NFR-1 is served by C-1, which E-1 realises; cut that link to isolate E-2.
components = [{"id": "C-1", "name": "Ingress", "requirements": ["FR-1"]}]
for version in (before, after):
version["technical_decomposition"]["attachments"] = {
"components.json": json.dumps(components)
}
summary = change_summary(before, after)
assert summary["checks"]["changed"] == ["K-2"]
assert summary["unrequested"] == {"checks": ["K-2"]}
def test_added_and_removed_items_are_found_by_id():
from tests.reference_workflow.changes import change_summary
def grow(graph):
graph["elements"].append(
{
"id": "E-3",
"name": "Second region service",
"component": "C-1",
"product": "Example Run",
"resource_type": "example_run_service",
}
)
graph["checks"] = [graph["checks"][0]] + [
{"id": "K-3", "applies_to": "E-3", "statement": "x", "source": "NFR-1"}
]
summary = change_summary(*versions(grow))
assert summary["elements"]["added"] == ["E-3"]
assert summary["checks"] == {"added": ["K-3"], "changed": [], "removed": ["K-2"]}
# K-2 came from DR-1 on E-2: nothing that changed reaches it.
assert summary["unrequested"] == {"checks": ["K-2"]}
# --- step 5: Approvals show the change --------------------------------------------------
async def test_an_approval_in_a_change_request_shows_what_that_phase_changes(
tmp_path,
):
from tests.reference_workflow.presentation import describe_prompt
await delivered(tmp_path)
engine = engine_for(tmp_path, changed())
first = await engine.start(
"Availability goes up.", customer_case=CASE, execution_id="v2", baseline="v1"
)
approval = first.prompt["approval"]
assert approval["phase"] == "requirements_discovery"
assert approval["baseline"] == "v1"
assert [item["id"] for item in approval["changes"]["requirements"]] == ["NFR-1"]
text = describe_prompt(first.prompt)
assert "This changes the architecture delivered in v1." in text
assert (
"- Requirement NFR-1 changed: 99.99% availability. (The service is now "
"customer-facing.)"
) in text
second = await engine.resume(
"v2", {"approval": {"phase": "requirements_discovery", "decision": "approve"}}
)
# The design did not change, and the Approval says so.
design = second.prompt["approval"]
assert design["phase"] == "solution_architecture" and design["changes"] == {}
assert "Nothing in this Phase differs from it." in describe_prompt(second.prompt)
async def test_an_ordinary_approval_says_nothing_of_changes(tmp_path):
first = await engine_for(tmp_path, once()).start("Ingest webhooks")
assert "changes" not in first.prompt["approval"]
# --- step 6: the change summary and the line of versions ----------------------------------
async def test_a_change_request_writes_changes_json_and_a_full_set_of_files(tmp_path):
from tests.reference_workflow.changes import change_summary
await delivered(tmp_path)
result = await engine_for(tmp_path, changed()).start(
"Availability goes up.",
customer_case=CASE,
pre_approved=True,
execution_id="v2",
baseline="v1",
)
case = tmp_path / "deliverables" / "cases" / CASE / "executions"
written = json.loads((case / "v2" / "changes.json").read_text())
assert [item["id"] for item in written["requirements"]] == ["NFR-1"]
assert written["unrequested"] == {}
# It matches a comparison of the two Executions' data files.
assert written == change_summary(
load_baseline(tmp_path, CASE, "v1")["deliverables"],
result.state["deliverables"],
)
# The Change request stands on its own: every Deliverable is written in full.
manifest = json.loads((case / "v2" / "execution.json").read_text())
assert manifest["changes"] == "changes.json"
assert set(manifest["deliverables"]) == {kind.value for kind in DEPENDENCY_GRAPH}
assert (case / "v2" / "solution-architecture-guide.md").is_file()
assert not (case / "v1" / "changes.json").exists()
# --- only the pillars a change touches are consulted again ------------------------------
async def test_a_change_request_starts_from_the_baselines_pillar_advice(tmp_path):
"""The design advisor is told which pillars the change affects, and has the
Baseline's advice for the rest."""
from execution_engine.adk.fake_runtime import FakeDecider
from execution_engine.adk.runtime import Answer
from tests.reference_workflow.pillars import ADVICE_KEY
await delivered(tmp_path)
# A Baseline whose design advisor consulted two pillars.
folder = tmp_path / "deliverables" / "cases" / CASE / "executions" / "v1"
advice = {"security": "Use a dedicated identity.", "reliability": "Two zones."}
(folder / "working.json").write_text(
json.dumps({"design_recommendations": {ADVICE_KEY: advice}})
)
seen: dict = {}
def advisor(task):
seen["pillars_to_consult"] = task.context["pillars_to_consult"]
return [sample_submission(K.DESIGN_RECOMMENDATIONS)]
decider = FakeDecider(
{"security": Answer(0.3, 0.8), "reliability": Answer(1.9, 0.9)}
)
runtime = FakeAgentRuntime(changed(**{"design-advisor": [advisor]}))
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
decider=decider,
)
result = await engine.start(
"Availability goes up.",
customer_case=CASE,
pre_approved=True,
execution_id="v2",
baseline="v1",
)
assert seen["pillars_to_consult"] == ["reliability"]
state, _, _ = decider.asked[0]
assert "Requirement NFR-1 changed: 99.99% availability." in state
# The advice is carried into the new Execution, and written for the next change.
assert result.state["working"]["design_recommendations"][ADVICE_KEY] == advice
assert json.loads((folder.parent / "v2" / "working.json").read_text()) == {
"design_recommendations": {ADVICE_KEY: advice}
}
async def test_a_first_consultation_is_given_no_hint(tmp_path):
seen: dict = {}
def advisor(task):
seen["pillars_to_consult"] = task.context["pillars_to_consult"]
return [sample_submission(K.DESIGN_RECOMMENDATIONS)]
from execution_engine.adk.fake_runtime import FakeDecider
decider = FakeDecider({})
turns = once()
turns["design-advisor"] = [advisor]
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=FakeAgentRuntime(turns),
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
decider=decider,
)
await engine.start("Ingest webhooks", pre_approved=True)
assert seen["pillars_to_consult"] is None and decider.asked == []
# --- changed, or only reworded? ---------------------------------------------------------
async def test_an_item_that_was_only_reworded_is_listed_apart_from_real_changes():
from execution_engine.adk.fake_runtime import FakeDecider
from execution_engine.adk.runtime import Answer
from tests.reference_workflow.changes import change_summary
from tests.reference_workflow.meaning import settle_meaning
def revise(graph):
graph["checks"][0]["statement"] = "The service runs at least three instances."
graph["checks"][1]["statement"] = "This service alone uses the identity."
before, after = versions(revise)
summary = change_summary(before, after)
assert summary["checks"]["changed"] == ["K-1", "K-2"]
decider = FakeDecider({"K-1": Answer(0.95, 0.9), "K-2": Answer(0.1, 0.8)})
settled = await settle_meaning(decider, summary, before, after, scope="v2")
assert settled["checks"]["changed"] == ["K-1"]
assert settled["checks"]["reworded"] == ["K-2"]
assert settled["meaning"] == {"K-1": 0.95, "K-2": 0.1}
# K-2 was outside the reach of what changed; reworded, it is no longer unrequested.
assert settled["unrequested"] == {}
_, questions, scope = decider.asked[0]
assert scope == "v2"
assert "The identity is used by this service only." in questions["K-2"].instructions
assert "This service alone uses the identity." in questions["K-2"].instructions
# The summary given in is not altered.
assert summary["checks"]["changed"] == ["K-1", "K-2"]
async def test_without_a_decision_everything_that_differs_stays_changed():
from tests.reference_workflow.changes import change_summary
from tests.reference_workflow.meaning import settle_meaning
def revise(graph):
graph["checks"][1]["statement"] = "This service alone uses the identity."
before, after = versions(revise)
summary = change_summary(before, after)
settled = await settle_meaning(NoDecider(), summary, before, after)
assert settled["checks"]["changed"] == ["K-2"]
assert "reworded" not in settled["checks"] and "meaning" not in settled

View File

@@ -0,0 +1,446 @@
"""Two savings on a Revision of the design.
A coverage review at submission finds what validation would otherwise find later,
at the cost of a Revision. And a Deliverable written from the shape of the design is
not written again when a Revision only changes its Checks.
"""
import json
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime, FakeDecider
from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec, Submitted
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from execution_engine.tools.execution import specialist_tools
from tests.reference_workflow.coverage import CoverageReview
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
SAMPLE_DATA,
WORKFLOW,
DeliverableKind,
sample_submission,
)
K = DeliverableKind
GRAPH = {"target-architecture.json": SAMPLE_DATA["target-architecture.json"]}
UPSTREAM = {
"requirements": {
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
},
"technical_decomposition": {
"attachments": {"components.json": SAMPLE_DATA["components.json"]}
},
"product_selection": {
"attachments": {"products.json": SAMPLE_DATA["products.json"]}
},
"design_recommendations": {
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
},
}
CONTENT = "## Design rationale\nReasons.\n"
# --- the review ------------------------------------------------------------------------
def choice(option: str, probability: float) -> Answer:
return Answer(option, probability, {option: probability})
async def test_the_review_asks_which_check_verifies_each_condition():
decider = FakeDecider({"NFR-1#1": choice("K-1", 0.9), "DR-1#1": choice("K-2", 0.9)})
concerns = await CoverageReview(decider)(
K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1"
)
state, questions, scope = decider.asked[0]
assert concerns == ()
assert scope == "exec-1"
assert "E-1 Ingress service (Example Run, example_run_service)" in state
# Constraints only: a functional requirement is carried by its component.
assert set(questions) == {"NFR-1#1", "DR-1#1"}
question = questions["NFR-1#1"]
assert question.kind == "choice" and question.options == ("K-1", "none")
assert 'Condition: "The service stays available when one instance fails."' in (
question.instructions
)
assert "- K-1 on Ingress service: The service runs at least two instances." in (
question.instructions
)
async def test_a_condition_no_check_verifies_is_raised_with_the_checks_it_has():
decider = FakeDecider({"NFR-1#1": choice("none", 0.9), "DR-1#1": choice("K-2", 1)})
concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM)
condition = "The service stays available when one instance fails."
assert concerns == (f'NFR-1: no Check verifies "{condition}" (its Checks: K-1)',)
async def test_an_uncertain_none_is_not_raised():
decider = FakeDecider({"NFR-1#1": choice("none", 0.45), "DR-1#1": choice("K-2", 1)})
review = CoverageReview(decider, none_at=0.6)
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
async def test_each_condition_of_a_requirement_is_asked_about_separately():
requirements = json.loads(SAMPLE_DATA["requirements.json"])
requirements[1]["conditions"] = [
"Survives one instance failing.",
"Survives a zone failing.",
]
upstream = {
**UPSTREAM,
"requirements": {
"attachments": {"requirements.json": json.dumps(requirements)}
},
}
decider = FakeDecider(
{
"NFR-1#1": choice("K-1", 0.9),
"NFR-1#2": choice("none", 0.8),
"DR-1#1": choice("K-2", 1),
}
)
concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream)
assert len(concerns) == 1 and "Survives a zone failing." in concerns[0]
async def test_an_item_without_conditions_is_read_as_one_condition_its_text():
requirements = json.loads(SAMPLE_DATA["requirements.json"])
del requirements[1]["conditions"]
upstream = {
**UPSTREAM,
"requirements": {
"attachments": {"requirements.json": json.dumps(requirements)}
},
}
decider = FakeDecider({"NFR-1#1": choice("K-1", 1), "DR-1#1": choice("K-2", 1)})
await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream)
assert 'Condition: "99.9% availability."' in (
decider.asked[0][1]["NFR-1#1"].instructions
)
async def test_no_decision_and_other_deliverables_raise_nothing():
review = CoverageReview(NoDecider())
decider = FakeDecider({})
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
assert await CoverageReview(decider)(K.REQUIREMENTS, {}, UPSTREAM) == ()
assert await CoverageReview(decider)(K.TARGET_ARCHITECTURE, {}, UPSTREAM) == ()
assert decider.asked == []
async def test_a_recommendation_marked_not_applicable_is_not_reviewed():
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
graph["checks"][1]["source"] = "NFR-1"
graph["not_applicable"] = [{"recommendation": "DR-1", "reason": "No identity."}]
decider = FakeDecider({"NFR-1#1": choice("K-1", 1)})
await CoverageReview(decider)(
K.TARGET_ARCHITECTURE,
{"target-architecture.json": json.dumps(graph)},
UPSTREAM,
)
assert set(decider.asked[0][1]) == {"NFR-1#1"}
# --- at submission ---------------------------------------------------------------------
def submit_with(review, working=None):
(submit,) = (
tool
for tool in specialist_tools(
WORKFLOW,
K.TARGET_ARCHITECTURE,
(),
allow_cloud_read=False,
upstream=UPSTREAM,
working={} if working is None else working,
review=review,
)
if tool.name == "submit_deliverable"
)
return submit.handler
async def test_what_the_review_raises_is_returned_once_and_then_accepted():
calls = []
async def review(attachments):
calls.append(attachments)
return ['NFR-1: no Check verifies "Two instances." (its Checks: K-1)']
working: dict = {}
submit = submit_with(review, working)
first = await submit(content=CONTENT, attachments=dict(GRAPH))
second = await submit(content=CONTENT)
assert first.outcome is None
assert first.message.startswith("Not submitted. A review found")
assert "NFR-1: no Check verifies" in first.message
assert "runs once" in first.message
# The graph it sent is kept, so it amends that and does not send it again.
assert isinstance(second.outcome, Submitted)
assert len(calls) == 1
async def test_a_structural_fault_is_reported_before_any_review():
async def review(attachments):
raise AssertionError("not reached")
result = await submit_with(review)(content=CONTENT, attachments={})
assert "attachment: target-architecture.json" in result.message
async def test_a_review_that_fails_or_raises_nothing_never_stops_a_submission():
async def broken(attachments):
raise RuntimeError("service down")
async def quiet(attachments):
return ()
for review in (broken, quiet):
result = await submit_with(review)(content=CONTENT, attachments=dict(GRAPH))
assert isinstance(result.outcome, Submitted)
# --- a description stands while the shape of the design does ---------------------------
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def graph_with(**changes) -> Call:
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
for key, change in changes.items():
graph[key] = change(graph[key])
return Call(
"submit_deliverable",
{
"content": CONTENT,
"attachments": {"target-architecture.json": json.dumps(graph)},
},
)
def addition() -> Call:
"""A validation that passes every Check and proposes one more."""
call = sample_submission(K.VALIDATION_RESULTS)
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
verdicts["proposed_additions"] = [
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": "NFR-1"}
]
return Call(
call.tool,
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
)
async def run(tmp_path, second_graph: Call, **options):
turns = {
spec.specialist: [[sample_submission(kind)]] * 2
for kind, spec in DEPENDENCY_GRAPH.items()
}
turns["target-architect"] = [
[sample_submission(K.TARGET_ARCHITECTURE)],
[second_graph],
]
turns["validator"] = [[addition()], [sample_submission(K.VALIDATION_RESULTS)]]
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
**options,
)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
return result, [run.role for run in runtime.runs]
async def test_the_description_is_kept_when_a_revision_only_changes_checks(tmp_path):
reworded = graph_with(
checks=lambda checks: [
{**checks[0], "statement": "min_instance_count is at least 2"},
checks[1],
]
)
result, roles_run = await run(tmp_path, reworded)
assert roles_run.count("target-architect") == 2
assert roles_run.count("iac-author") == 2 # the code answers the Checks
assert roles_run.count("describer") == 1
# The Ledger still records a second submission: the Deliverable was not skipped.
submitted = [
event
for event in result.state["ledger"]["events"]
if event["type"] == "submitted"
and event["subject"] == "architecture_description"
]
assert len(submitted) == 2
async def test_the_description_is_written_again_when_the_shape_changes(tmp_path):
renamed = graph_with(
elements=lambda elements: [{**elements[0], "name": "Edge service"}, elements[1]]
)
_, roles_run = await run(tmp_path, renamed)
assert roles_run.count("describer") == 2
async def test_the_engine_reviews_a_design_before_it_is_submitted(tmp_path):
raised = []
async def review(kind, attachments, upstream, scope=""):
if kind == K.TARGET_ARCHITECTURE and not raised:
raised.append(scope)
return ['NFR-1 "Two instances." (its Checks: K-1)']
return ()
turns = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
turns["target-architect"] = [
[sample_submission(K.TARGET_ARCHITECTURE)] * 2 # refused once, then accepted
]
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
submission_review=review,
)
result = await engine.start("Ingest webhooks", pre_approved=True, execution_id="e1")
assert result.status == "completed"
assert raised == ["e1"]
architect = next(run for run in runtime.runs if run.role == "target-architect")
assert architect.results[0][1].startswith("Not submitted. A review found")
# What a review raises is kept as a lesson candidate, like any refusal.
assert any(
"A review found" in json.dumps(outcome.get("refusals"))
for outcome in result.state["outcomes"]
)
async def test_feedback_on_the_description_itself_has_it_written_again(tmp_path):
"""Kept only while nobody has asked for the description to change."""
from execution_engine.nodes.execution_nodes import STANDING, make_execution_nodes
nodes = make_execution_nodes(
workflow=WORKFLOW,
runtime=FakeAgentRuntime(
{"describer": [[sample_submission(K.ARCHITECTURE_DESCRIPTION, "new")]]}
),
roles=ROLES,
recorder=InMemoryRecorder(),
output_root=tmp_path,
)
upstream = {"target_architecture": {"attachments": dict(GRAPH)}}
from tests.reference_workflow.chain import graph_structure
standing = {
"basis": graph_structure(upstream),
"feedback": [],
"content": "old",
"attachments": {},
}
task = {
"execution_id": "e1",
"role": "describer",
"kind": "architecture_description",
"attempt": 2,
"allow_cloud_read": False,
"working": {STANDING: standing},
"context": {
"dependencies": upstream,
"feedback": ["Dependency x was revised: y"],
},
}
kept = await nodes.specialist_task(task)
asked = await nodes.specialist_task(
{**task, "context": {**task["context"], "feedback": ["Make the flow clearer."]}}
)
assert kept["outcomes"][0]["content"] == "old"
assert "new" in asked["outcomes"][0]["content"]
assert asked["working"]["architecture_description"][STANDING]["feedback"] == [
"Make the flow clearer."
]
# --- requirements name no products ------------------------------------------------------
async def test_a_requirement_that_names_a_product_is_raised():
from tests.reference_workflow.requirements_review import RequirementsReview
requirements = [
{"id": "FR-1", "kind": "functional", "text": "Run the API on Cloud Run."},
{"id": "FR-2", "kind": "functional", "text": "Accept signed webhooks."},
]
decider = FakeDecider({"FR-1": Answer(0.93, 0.86), "FR-2": Answer(0.05, 0.9)})
raised = await RequirementsReview(decider)(
K.REQUIREMENTS, {"requirements.json": json.dumps(requirements)}, None, "e1"
)
assert len(raised) == 1 and raised[0].startswith("FR-1 names a product")
_, questions, scope = decider.asked[0]
assert set(questions) == {"FR-1", "FR-2"} and scope == "e1"
assert "Accept signed webhooks." in questions["FR-2"].instructions
async def test_the_requirements_review_looks_only_at_requirements_and_never_blocks():
from tests.reference_workflow.requirements_review import RequirementsReview
decider = FakeDecider({})
review = RequirementsReview(decider)
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
assert await review(K.REQUIREMENTS, {"requirements.json": "not json"}, None) == ()
assert decider.asked == []
assert (
await RequirementsReview(NoDecider())(
K.REQUIREMENTS,
{"requirements.json": SAMPLE_DATA["requirements.json"]},
None,
)
== ()
)

View File

@@ -0,0 +1,310 @@
"""The dependency graph and Deliverable contracts."""
import pytest
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
PHASE_ORDER,
DeliverableKind,
PhaseId,
check_submission,
deliverables_in,
inputs_of,
)
def test_every_kind_is_declared_once_with_known_dependencies():
assert set(DEPENDENCY_GRAPH) == set(DeliverableKind)
for kind, spec in DEPENDENCY_GRAPH.items():
assert spec.kind is kind
assert set(spec.depends_on) <= set(DeliverableKind), kind
def test_dependencies_never_point_to_a_later_phase_and_form_no_cycle():
position = {phase: index for index, phase in enumerate(PHASE_ORDER)}
for spec in DEPENDENCY_GRAPH.values():
for dependency in spec.depends_on:
assert position[DEPENDENCY_GRAPH[dependency].phase] <= position[spec.phase]
visiting: set[str] = set()
done: set[str] = set()
def visit(kind: str) -> None:
assert kind not in visiting, f"cycle through {kind}"
if kind in done:
return
visiting.add(kind)
for dependency in DEPENDENCY_GRAPH[kind].depends_on:
visit(dependency)
visiting.discard(kind)
done.add(kind)
for kind in DeliverableKind:
visit(kind)
def test_every_phase_groups_at_least_one_deliverable_with_a_distinct_specialist():
assert PHASE_ORDER == (
PhaseId.SCOPING,
PhaseId.REQUIREMENTS_DISCOVERY,
PhaseId.SOLUTION_ARCHITECTURE,
PhaseId.SOLUTION_IMPLEMENTATION,
PhaseId.SOLUTION_VALIDATION,
PhaseId.PACKAGING,
)
for phase in PHASE_ORDER:
assert deliverables_in(phase), phase
specialists = [spec.specialist for spec in DEPENDENCY_GRAPH.values()]
assert len(specialists) == len(set(specialists))
def test_the_target_architecture_sits_between_the_design_and_the_code():
depends = {kind: set(spec.depends_on) for kind, spec in DEPENDENCY_GRAPH.items()}
K = DeliverableKind
assert depends[K.SCOPE] == set()
assert depends[K.REQUIREMENTS] == {K.SCOPE}
assert depends[K.TECHNICAL_DECOMPOSITION] == {K.REQUIREMENTS}
assert depends[K.PRODUCT_SELECTION] == {K.TECHNICAL_DECOMPOSITION}
assert depends[K.DESIGN_RECOMMENDATIONS] == {K.PRODUCT_SELECTION, K.REQUIREMENTS}
assert depends[K.TARGET_ARCHITECTURE] == {
K.PRODUCT_SELECTION,
K.DESIGN_RECOMMENDATIONS,
}
# Everything after the graph is derived from it.
assert depends[K.ARCHITECTURE_DESCRIPTION] == {K.TARGET_ARCHITECTURE}
assert depends[K.DEPLOYMENT_GUIDANCE] == {K.TARGET_ARCHITECTURE}
assert depends[K.VALIDATION_RESULTS] == {K.DEPLOYMENT_GUIDANCE}
assert depends[K.SOLUTION_GUIDE] == set(
deliverables_in(PhaseId.SOLUTION_VALIDATION)
)
def test_the_design_is_approved_before_code_is_written():
design = PhaseId.SOLUTION_ARCHITECTURE
implementation = PhaseId.SOLUTION_IMPLEMENTATION
assert DEPENDENCY_GRAPH[DeliverableKind.TARGET_ARCHITECTURE].phase is design
assert DEPENDENCY_GRAPH[DeliverableKind.DEPLOYMENT_GUIDANCE].phase is implementation
assert PHASE_ORDER.index(design) < PHASE_ORDER.index(implementation)
def test_inputs_are_earlier_phases_plus_indirect_dependencies_in_graph_order():
design = (
DeliverableKind.SCOPE,
DeliverableKind.REQUIREMENTS,
DeliverableKind.TECHNICAL_DECOMPOSITION,
DeliverableKind.PRODUCT_SELECTION,
DeliverableKind.DESIGN_RECOMMENDATIONS,
DeliverableKind.TARGET_ARCHITECTURE,
)
assert inputs_of(DeliverableKind.SCOPE) == ()
assert inputs_of(DeliverableKind.REQUIREMENTS) == (DeliverableKind.SCOPE,)
assert inputs_of(DeliverableKind.DEPLOYMENT_GUIDANCE) == design
assert inputs_of(DeliverableKind.ARCHITECTURE_DESCRIPTION) == design
assert inputs_of(DeliverableKind.TARGET_ARCHITECTURE) == design[:5]
# The plan is produced beside the guide, not before it: neither reads the other.
assert set(inputs_of(DeliverableKind.SOLUTION_GUIDE)) == set(DeliverableKind) - {
DeliverableKind.SOLUTION_GUIDE,
DeliverableKind.WORK_BREAKDOWN,
}
assert DeliverableKind.SOLUTION_GUIDE not in inputs_of(
DeliverableKind.WORK_BREAKDOWN
)
def sections_missing(kind: DeliverableKind, content: str) -> list[str]:
return [m for m in check_submission(kind, content, {}) if m.startswith("section:")]
def test_submission_meeting_the_contract_has_nothing_missing():
content = (
"# Deployment guidance\n"
"## 6.1. Deployment prerequisites\nEnable APIs.\n"
"## 6.2. Step-by-step deployment instructions\nRun terraform.\n"
)
attachments = {
"terraform/main.tf": "resource {}",
"terraform/variables.tf": "variable {}",
"implementation.json": "[]",
}
assert (
check_submission(DeliverableKind.DEPLOYMENT_GUIDANCE, content, attachments)
== ()
)
def test_submission_reports_missing_sections_and_attachments():
missing = check_submission(
DeliverableKind.DEPLOYMENT_GUIDANCE,
"## Deployment prerequisites\n",
{"terraform/main.tf": "resource {}"},
)
assert missing == (
"section: Step-by-step deployment instructions",
"attachment: terraform/variables.tf",
"attachment: implementation.json",
)
@pytest.mark.parametrize(
"heading",
[
"### 5.1. Security, privacy, and compliance",
"## security, privacy, and compliance",
"# 5.1 Security, Privacy, and Compliance:",
],
)
def test_section_matching_ignores_numbering_case_and_trailing_punctuation(heading):
content = (
f"{heading}\n"
"## Reliability\n"
"## Operational excellence\n"
"## Cost optimization\n"
"## Performance efficiency\n"
"## Sustainability\n"
)
assert sections_missing(DeliverableKind.DESIGN_RECOMMENDATIONS, content) == []
def test_section_matches_a_provider_prefixed_heading_but_not_a_longer_word():
content = (
"### 4.1. Google Cloud products and features mapping\n"
"## Non-functional requirements\n"
)
assert sections_missing(DeliverableKind.PRODUCT_SELECTION, content) == []
assert "section: Functional requirements" in check_submission(
DeliverableKind.REQUIREMENTS, content, {}
)
def test_section_names_in_body_text_do_not_count_as_headings():
missing = check_submission(
DeliverableKind.REQUIREMENTS,
"Functional requirements, non-functional requirements, current state and "
"dependencies are all covered below.\n",
{},
)
assert "section: Functional requirements" in missing
def test_empty_content_is_refused():
assert check_submission(DeliverableKind.TECHNICAL_DECOMPOSITION, " \n", {}) == (
"content: must not be empty",
)
# --- contracts beyond headings ---------------------------------------------------------
def test_architecture_description_needs_component_and_flow_sections():
missing = check_submission(
DeliverableKind.ARCHITECTURE_DESCRIPTION, "The system works.", {}
)
assert missing == ("section: Components", "section: Flow")
assert (
check_submission(
DeliverableKind.ARCHITECTURE_DESCRIPTION,
"## Architecture components\nA.\n## Data flow\nB.\n",
{},
)
== ()
)
def test_a_required_attachment_must_have_content():
missing = check_submission(
DeliverableKind.VALIDATION_RESULTS,
"## Deployment dry-run\nPass.\n## Architecture and policy analysis\nPass.\n",
{"verdicts.json": " \n"},
)
assert missing == ("attachment: verdicts.json must not be empty",)
# --- what a Specialist is shown of what it is given --------------------------------------
def given_for(kind: DeliverableKind) -> dict:
from tests.reference_workflow.workflow import dependencies_for, sample_submission
submitted = {}
for each in DeliverableKind:
call = sample_submission(each)
submitted[each.value] = {
"content": call.args["content"],
"attachments": call.args["attachments"],
}
review = "\n## Architecture framework checklist review\n| a | b |\n"
submitted["design_recommendations"]["content"] += review
return dependencies_for(kind, submitted)
def test_a_specialist_is_shown_only_what_it_uses():
from tests.reference_workflow.workflow import shown
author = shown(
DeliverableKind.DEPLOYMENT_GUIDANCE,
given_for(DeliverableKind.DEPLOYMENT_GUIDANCE),
)
validator = shown(
DeliverableKind.VALIDATION_RESULTS,
given_for(DeliverableKind.VALIDATION_RESULTS),
)
# The graph is all the code is written from.
assert list(author) == ["target_architecture"]
# The validator judges code against Checks, and reads requirements and
# recommendations for coverage; the prose describing the design is not judged.
assert "architecture_description" not in validator
assert {"requirements", "design_recommendations", "deployment_guidance"} <= set(
validator
)
def test_nothing_is_taken_from_what_a_submission_is_checked_against():
"""Showing less changes what the model reads, never what the code checks."""
given = given_for(DeliverableKind.DEPLOYMENT_GUIDANCE)
assert {"requirements", "design_recommendations", "scope"} <= set(given)
def test_the_scope_is_shown_once_and_only_its_prose_to_who_works_from_it():
from tests.reference_workflow.workflow import shown
analyst = shown(
DeliverableKind.REQUIREMENTS, given_for(DeliverableKind.REQUIREMENTS)
)
architect = shown(
DeliverableKind.TARGET_ARCHITECTURE,
given_for(DeliverableKind.TARGET_ARCHITECTURE),
)
assert list(analyst["scope"]) == ["content"]
assert "scope" not in architect
def test_a_section_written_for_a_reader_is_left_off_except_for_the_guide():
from tests.reference_workflow.pillars import REVIEW_HEADING
from tests.reference_workflow.workflow import FOR_READERS, shown
assert FOR_READERS[DeliverableKind.DESIGN_RECOMMENDATIONS] == REVIEW_HEADING
architect = shown(
DeliverableKind.TARGET_ARCHITECTURE,
given_for(DeliverableKind.TARGET_ARCHITECTURE),
)
packager = shown(
DeliverableKind.SOLUTION_GUIDE, given_for(DeliverableKind.SOLUTION_GUIDE)
)
assert REVIEW_HEADING not in architect["design_recommendations"]["content"]
assert "## Design" in architect["design_recommendations"]["content"] or architect[
"design_recommendations"
]["content"].startswith("# design_recommendations")
assert REVIEW_HEADING in packager["design_recommendations"]["content"]

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,78 @@
"""Files cross tool calls as a list of {path, content}: models rewrite object keys."""
import pytest
from execution_engine.adk.runtime import FILES_SCHEMA, Submitted, files_from_argument
from execution_engine.tools.execution import specialist_tools
from tests.reference_workflow.local import TerraformRunner, local_tools
from tests.reference_workflow.workflow import WORKFLOW, DeliverableKind
GUIDANCE = (
"# Deployment prerequisites\n\nx\n\n# Step-by-step deployment instructions\n\ny\n"
)
def test_files_are_offered_as_a_list_of_path_and_content():
assert FILES_SCHEMA["type"] == "array"
assert FILES_SCHEMA["items"]["required"] == ["path", "content"]
def test_file_arguments_accept_lists_and_maps():
as_list = [{"path": "terraform/main.tf", "content": "a"}]
assert files_from_argument(as_list) == {"terraform/main.tf": "a"}
assert files_from_argument({"terraform/main.tf": "a"}) == {"terraform/main.tf": "a"}
assert files_from_argument(None) == {}
with pytest.raises(TypeError):
files_from_argument(["terraform/main.tf"])
async def test_submission_takes_attachments_as_a_list():
submit = specialist_tools(
WORKFLOW, DeliverableKind.DEPLOYMENT_GUIDANCE, extra=[], allow_cloud_read=False
)[0]
result = await submit.handler(
content=GUIDANCE,
attachments=[
{"path": "terraform/main.tf", "content": "resource {}"},
{"path": "terraform/variables.tf", "content": "variable {}"},
{"path": "implementation.json", "content": "[]"},
],
)
assert result.outcome == Submitted(
GUIDANCE,
{
"terraform/main.tf": "resource {}",
"terraform/variables.tf": "variable {}",
"implementation.json": "[]",
},
)
async def test_malformed_attachments_are_refused_to_the_agent():
submit = specialist_tools(
WORKFLOW, DeliverableKind.DEPLOYMENT_GUIDANCE, extra=[], allow_cloud_read=False
)[0]
result = await submit.handler(content=GUIDANCE, attachments=["terraform/main.tf"])
assert result.outcome is None
assert result.message.startswith("Refused: attachments")
async def test_terraform_tools_take_files_as_a_list():
received = {}
class Runner(TerraformRunner):
async def validate(self, files):
received.update(files)
return await super().validate({})
(validate,) = local_tools(["terraform_validate"], Runner())
await validate.handler(files=[{"path": "terraform/main.tf", "content": "x"}])
assert validate.parameters["properties"]["files"]["type"] == "array"
assert received == {"terraform/main.tf": "x"}

View File

@@ -0,0 +1,172 @@
"""Intake: what a message that starts something is asking for."""
import json
from execution_engine.adk.fake_runtime import FakeDecider
from execution_engine.adk.runtime import Answer, NoDecider
from execution_engine.nodes.intake import CHANGE_REQUEST, NEW_REQUEST, QUESTION, Intake
from tests.reference_workflow.workflow import WORKFLOW
def delivered(tmp_path, case: str, statuses: list[str]) -> None:
folder = tmp_path / "deliverables" / "cases" / case / "intent"
folder.mkdir(parents=True)
intents = [
{
"version": n,
"workflow_request": f"Design thing {n}.",
"status": status,
"execution_id": f"v{n}",
}
for n, status in enumerate(statuses, start=1)
]
(folder / "history.json").write_text(
json.dumps({"customer_case": case, "intents": intents})
)
async def test_without_a_decision_every_message_is_a_new_request(tmp_path):
reading = await Intake(NoDecider(), tmp_path).read("What can you do?")
assert reading.kind == NEW_REQUEST and not reading.judged
async def test_a_confident_question_is_read_as_one(tmp_path):
decider = FakeDecider({"intake": Answer(QUESTION, 0.98)})
reading = await Intake(decider, tmp_path).read("How long does it take?", scope="t1")
state, questions, scope = decider.asked[0]
assert reading.kind == QUESTION and reading.judged
assert scope == "t1"
assert questions["intake"].options == (NEW_REQUEST, CHANGE_REQUEST, QUESTION)
assert state.endswith("The user's message: How long does it take?")
async def test_a_reading_that_is_not_sure_enough_starts_new_work(tmp_path):
decider = FakeDecider({"intake": Answer(QUESTION, 0.6)})
assert (await Intake(decider, tmp_path, act=0.75).read("hello")).kind == NEW_REQUEST
async def test_a_change_needs_something_delivered_to_change(tmp_path):
decider = FakeDecider({"intake": Answer(CHANGE_REQUEST, 0.99)})
delivered(tmp_path, "acme", ["completed", "in_progress"])
intake = Intake(decider, tmp_path)
with_history = await intake.read("Raise availability to 99.99%.", "acme")
without = await intake.read("Raise availability to 99.99%.", "new-client")
no_case = await intake.read("Raise availability to 99.99%.")
assert with_history.kind == CHANGE_REQUEST and with_history.earlier == 1
assert with_history.latest == "v1" # what the change would change
assert "- Design thing 1." in decider.asked[0][0]
assert "Design thing 2." not in decider.asked[0][0] # still in progress
assert without.kind == NEW_REQUEST and no_case.kind == NEW_REQUEST
# --- a question about what was delivered ------------------------------------------------
async def test_a_question_about_delivered_work_is_told_from_one_about_the_service(
tmp_path,
):
delivered(tmp_path, "acme", ["completed"])
about = FakeDecider(
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.9, 0.9)}
)
general = FakeDecider(
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.1, 0.9)}
)
first = await Intake(about, tmp_path).read("Why Cloud SQL?", "acme")
second = await Intake(general, tmp_path).read("What can you do?", "acme")
assert first.kind == QUESTION and first.about_delivered and first.latest == "v1"
assert second.kind == QUESTION and not second.about_delivered
async def test_with_nothing_delivered_a_question_is_never_about_delivered_work(
tmp_path,
):
decider = FakeDecider(
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.99, 0.9)}
)
reading = await Intake(decider, tmp_path).read("Why Cloud SQL?", "new-client")
assert not reading.about_delivered
assert len(decider.asked) == 1 # the second question was not asked
async def test_an_answer_is_written_from_the_delivered_guide_and_nothing_else(
tmp_path,
):
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import (
Call,
FakeAgentRuntime,
)
from execution_engine.adk.runtime import RoleSpec
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.intake import DeliveredAnswers
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
sample_submission,
)
roles = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
seen: dict = {}
def writer(task):
seen.update(task.context)
return [Call("write_reply", {"text": "Cloud SQL, for relational bookings."})]
runtime = FakeAgentRuntime(
{
**{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
},
"reply-writer": [writer],
}
)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
await engine.start(
"Design bookings.", customer_case="acme", pre_approved=True, execution_id="v1"
)
runs_before = len(runtime.runs)
answers = DeliveredAnswers(
runtime,
RoleSpec("reply-writer", "Write.", budget=3),
tmp_path,
"solution_guide",
)
answer = await answers.answer("Why Cloud SQL?", "acme", "v1")
assert answer == "Cloud SQL, for relational bookings."
assert seen["question"] == "Why Cloud SQL?"
assert seen["delivered"]["execution_id"] == "v1"
assert seen["delivered"]["request"] == "Design bookings."
assert seen["delivered"]["final_document"]
assert len(runtime.runs) == runs_before + 1 # one writer run; no Execution
# An Execution that is not there gives no answer, not an error.
assert await answers.answer("Why?", "acme", "missing") is None

View File

@@ -0,0 +1,397 @@
"""The Ledger: Dependency acceptance, Approvals and Revision rules."""
import pytest
from execution_engine.states.ledger import (
DeliverableStatus,
Ledger,
LedgerRefusal,
PhaseStatus,
)
from tests.reference_workflow.workflow import WORKFLOW
from tests.reference_workflow.workflow import DeliverableKind as K
from tests.reference_workflow.workflow import PhaseId as P
def produce(ledger: Ledger, *kinds: K) -> None:
"""Delegate and submit each kind, in order."""
for kind in kinds:
attempt = ledger.start([kind], actor="orchestrator")[kind]
ledger.submit(kind, attempt, actor=f"specialist:{kind}")
def approve(ledger: Ledger, phase: P) -> None:
ledger.request_approval(phase, actor="orchestrator")
if ledger.phase_status(phase) is PhaseStatus.AWAITING_APPROVAL:
ledger.approve(phase, actor="user")
def scoped(**options) -> Ledger:
"""A Ledger whose scope is agreed: where requirements discovery begins."""
ledger = Ledger(WORKFLOW, **options)
produce(ledger, K.SCOPE)
approve(ledger, P.SCOPING)
return ledger
def through_phase_one(ledger: Ledger) -> None:
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
approve(ledger, P.REQUIREMENTS_DISCOVERY)
def through_phase_two(ledger: Ledger) -> None:
through_phase_one(ledger)
produce(
ledger, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE
)
approve(ledger, P.SOLUTION_ARCHITECTURE)
produce(ledger, K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)
approve(ledger, P.SOLUTION_IMPLEMENTATION)
def through_phase_three(ledger: Ledger) -> None:
through_phase_two(ledger)
produce(ledger, K.VALIDATION_RESULTS)
approve(ledger, P.SOLUTION_VALIDATION)
# --- ready work and the two acceptance thresholds -------------------------------
def test_a_new_execution_has_only_the_scope_ready():
ledger = Ledger(WORKFLOW)
assert ledger.ready_work() == (K.SCOPE,)
assert all(ledger.phase_status(p) is PhaseStatus.OPEN for p in P)
def test_once_the_scope_is_approved_requirements_are_ready():
ledger = scoped()
assert ledger.ready_work() == (K.REQUIREMENTS,)
assert ledger.phase_status(P.SCOPING) is PhaseStatus.APPROVED
def test_same_phase_dependency_is_met_once_submitted():
ledger = scoped()
produce(ledger, K.REQUIREMENTS)
assert ledger.ready_work() == (K.TECHNICAL_DECOMPOSITION,)
def test_cross_phase_dependency_needs_the_earlier_phase_approved():
ledger = scoped()
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
assert K.PRODUCT_SELECTION not in ledger.ready_work()
approve(ledger, P.REQUIREMENTS_DISCOVERY)
assert ledger.ready_work() == (K.PRODUCT_SELECTION,)
def test_independent_work_becomes_ready_together():
ledger = scoped()
through_phase_one(ledger)
produce(
ledger, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE
)
# Nothing in implementation is ready until the design is approved.
assert ledger.ready_work() == ()
approve(ledger, P.SOLUTION_ARCHITECTURE)
assert set(ledger.ready_work()) == {
K.ARCHITECTURE_DESCRIPTION,
K.DEPLOYMENT_GUIDANCE,
}
def test_ready_phases_lists_open_phases_with_every_deliverable_submitted():
ledger = scoped()
produce(ledger, K.REQUIREMENTS)
assert ledger.phases_ready_for_approval() == ()
produce(ledger, K.TECHNICAL_DECOMPOSITION)
assert ledger.phases_ready_for_approval() == (P.REQUIREMENTS_DISCOVERY,)
# --- delegation and submission ---------------------------------------------------
def test_start_refuses_work_that_is_not_ready_and_changes_nothing():
ledger = scoped()
with pytest.raises(LedgerRefusal, match="technical_decomposition"):
ledger.start([K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION], actor="orchestrator")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.PENDING
def test_start_moves_ready_work_in_progress_and_returns_attempts():
ledger = scoped()
attempts = ledger.start([K.REQUIREMENTS], actor="orchestrator")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.IN_PROGRESS
assert attempts == {K.REQUIREMENTS: 1}
assert ledger.ready_work() == ()
def test_submit_requires_the_current_attempt():
ledger = scoped()
attempt = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
with pytest.raises(LedgerRefusal, match="attempt"):
ledger.submit(K.REQUIREMENTS, attempt + 1, actor="requirements-analyst")
ledger.submit(K.REQUIREMENTS, attempt, actor="requirements-analyst")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED
def test_late_submission_from_a_superseded_attempt_is_refused_after_restart():
ledger = scoped()
first = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
ledger.release(K.REQUIREMENTS, first, reason="needs input")
second = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
with pytest.raises(LedgerRefusal, match="attempt"):
ledger.submit(K.REQUIREMENTS, first, actor="requirements-analyst")
ledger.submit(K.REQUIREMENTS, second, actor="requirements-analyst")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED
def test_submit_refuses_work_not_in_progress():
ledger = scoped()
with pytest.raises(LedgerRefusal, match="in_progress"):
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
def test_release_returns_work_to_pending_without_a_revision():
ledger = scoped()
attempt = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
ledger.release(K.REQUIREMENTS, attempt, reason="needs input")
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.PENDING
assert ledger.revisions(K.REQUIREMENTS) == 0
assert ledger.ready_work() == (K.REQUIREMENTS,)
with pytest.raises(LedgerRefusal, match="attempt"):
ledger.submit(K.REQUIREMENTS, attempt, actor="requirements-analyst")
# --- approvals ---------------------------------------------------------------------
def test_request_approval_refuses_until_every_deliverable_is_submitted():
ledger = scoped()
produce(ledger, K.REQUIREMENTS)
with pytest.raises(LedgerRefusal, match="technical_decomposition"):
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.OPEN
def test_approval_moves_through_awaiting_to_approved_by_the_user():
ledger = scoped()
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
assert (
ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.AWAITING_APPROVAL
)
ledger.approve(P.REQUIREMENTS_DISCOVERY, actor="user")
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
assert ledger.events[-1].type == "approved"
assert ledger.events[-1].actor == "user"
def test_approve_refuses_a_phase_not_awaiting_approval():
ledger = scoped()
with pytest.raises(LedgerRefusal, match="awaiting_approval"):
ledger.approve(P.REQUIREMENTS_DISCOVERY, actor="user")
def test_pre_approved_ledger_grants_approval_immediately():
ledger = scoped(pre_approved=True)
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
assert [e.type for e in ledger.events[-2:]] == ["approval_requested", "approved"]
assert ledger.events[-1].actor == "pre-approved"
def test_request_changes_reopens_the_phase_for_further_work():
ledger = scoped()
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
ledger.request_changes(P.REQUIREMENTS_DISCOVERY, "Add the RTO target", actor="user")
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.OPEN
assert ledger.events[-1].type == "changes_requested"
assert ledger.events[-1].detail == "Add the RTO target"
def test_finish_is_allowed_only_once_packaging_is_approved():
ledger = scoped()
through_phase_three(ledger)
assert not ledger.can_finish()
produce(ledger, K.WORK_BREAKDOWN, K.SOLUTION_GUIDE)
approve(ledger, P.PACKAGING)
assert ledger.can_finish()
# --- revisions ---------------------------------------------------------------------
def test_reopen_cascades_to_dependents_and_withdraws_affected_approvals():
ledger = scoped()
through_phase_three(ledger)
result = ledger.reopen(
K.DEPLOYMENT_GUIDANCE, "terraform plan failed", actor="orchestrator"
)
assert result.escalation is None
# solution_guide was still pending, so its status doesn't change and it isn't reset.
assert set(result.reset) == {K.DEPLOYMENT_GUIDANCE, K.VALIDATION_RESULTS}
assert ledger.status(K.DEPLOYMENT_GUIDANCE) is DeliverableStatus.PENDING
assert ledger.status(K.VALIDATION_RESULTS) is DeliverableStatus.PENDING
assert ledger.status(K.PRODUCT_SELECTION) is DeliverableStatus.SUBMITTED
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
# Revising the code leaves the approved design standing.
assert ledger.phase_status(P.SOLUTION_ARCHITECTURE) is PhaseStatus.APPROVED
assert ledger.phase_status(P.SOLUTION_IMPLEMENTATION) is PhaseStatus.OPEN
assert ledger.phase_status(P.SOLUTION_VALIDATION) is PhaseStatus.OPEN
assert set(result.withdrawn_approvals) == {
P.SOLUTION_IMPLEMENTATION,
P.SOLUTION_VALIDATION,
}
assert ledger.revisions(K.DEPLOYMENT_GUIDANCE) == 1
assert ledger.revisions(K.VALIDATION_RESULTS) == 0
assert ledger.feedback(K.DEPLOYMENT_GUIDANCE) == ("terraform plan failed",)
assert ledger.feedback(K.VALIDATION_RESULTS) == (
"Dependency deployment_guidance was revised: terraform plan failed",
)
assert ledger.ready_work() == (K.DEPLOYMENT_GUIDANCE,)
def test_reopen_invalidates_attempts_of_dependents_already_in_progress():
ledger = scoped()
through_phase_one(ledger)
produce(ledger, K.PRODUCT_SELECTION)
attempt = ledger.start([K.DESIGN_RECOMMENDATIONS], actor="orchestrator")[
K.DESIGN_RECOMMENDATIONS
]
ledger.reopen(K.PRODUCT_SELECTION, "user prefers GKE", actor="orchestrator")
assert ledger.status(K.DESIGN_RECOMMENDATIONS) is DeliverableStatus.PENDING
with pytest.raises(LedgerRefusal, match="attempt"):
ledger.submit(K.DESIGN_RECOMMENDATIONS, attempt, actor="design-advisor")
def test_reopen_refuses_deliverables_that_were_never_submitted():
ledger = scoped()
with pytest.raises(LedgerRefusal, match="submitted"):
ledger.reopen(K.REQUIREMENTS, "why", actor="orchestrator")
def test_reopen_escalates_at_the_revision_cap_without_changing_state():
ledger = scoped(max_revisions=2)
produce(ledger, K.REQUIREMENTS)
for _ in range(2):
ledger.reopen(K.REQUIREMENTS, "again", actor="orchestrator")
produce(ledger, K.REQUIREMENTS)
result = ledger.reopen(K.REQUIREMENTS, "one more", actor="orchestrator")
assert result.reset == ()
assert result.escalation is not None
assert "requirements" in result.escalation
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED
assert ledger.revisions(K.REQUIREMENTS) == 2
assert ledger.events[-1].type == "escalated"
def test_user_can_override_the_revision_cap():
ledger = scoped(max_revisions=1)
produce(ledger, K.REQUIREMENTS)
ledger.reopen(K.REQUIREMENTS, "first", actor="orchestrator")
produce(ledger, K.REQUIREMENTS)
result = ledger.reopen(
K.REQUIREMENTS, "user insists", actor="user", override_cap=True
)
assert result.escalation is None
assert ledger.revisions(K.REQUIREMENTS) == 2
def test_default_revision_cap_is_three():
assert scoped().max_revisions == 3
# --- records and persistence -------------------------------------------------------
def test_transitions_are_recorded_as_ordered_events():
ledger = Ledger(WORKFLOW)
produce(ledger, K.SCOPE)
assert [(e.sequence, e.type, e.subject) for e in ledger.events] == [
(1, "started", "scope"),
(2, "submitted", "scope"),
]
def test_round_trips_through_plain_data():
ledger = scoped(max_revisions=5)
through_phase_one(ledger)
produce(ledger, K.PRODUCT_SELECTION)
ledger.reopen(K.PRODUCT_SELECTION, "cost", actor="user")
restored = Ledger.from_dict(WORKFLOW, ledger.to_dict())
assert restored.to_dict() == ledger.to_dict()
assert restored.ready_work() == ledger.ready_work()
assert restored.max_revisions == 5
assert restored.events == ledger.events
def test_revising_the_target_architecture_returns_the_code_to_draft():
ledger = scoped()
through_phase_three(ledger)
result = ledger.reopen(
K.TARGET_ARCHITECTURE, "NFR-1 has no Check on retention", actor="orchestrator"
)
assert set(result.reset) == {
K.TARGET_ARCHITECTURE,
K.ARCHITECTURE_DESCRIPTION,
K.DEPLOYMENT_GUIDANCE,
K.VALIDATION_RESULTS,
}
assert set(result.withdrawn_approvals) == {
P.SOLUTION_ARCHITECTURE,
P.SOLUTION_IMPLEMENTATION,
P.SOLUTION_VALIDATION,
}
# The code cannot be rewritten until the revised design is approved again.
produce(ledger, K.TARGET_ARCHITECTURE)
assert ledger.ready_work() == ()

View File

@@ -0,0 +1,232 @@
"""Lesson candidates: what an Execution keeps about what went wrong in it."""
import json
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.runtime import RoleSpec
from execution_engine.agent import ExecutionEngine
from execution_engine.recorder import InMemoryRecorder
from execution_engine.states.lessons import refusal
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
WORKFLOW,
DeliverableKind,
PhaseId,
sample_submission,
)
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
K = DeliverableKind
P = PhaseId
ROLES = {
"orchestrator": RoleSpec(
"orchestrator", "Coordinate.", budget=40, skills=("solution-architecture",)
),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
FAILED = {"check": "K-2", "evidence": "The identity is shared with another service."}
ADDITION = {
"statement": "The service sets a request timeout.",
"applies_to": "E-1",
"reason": "NFR-1 has no Check on timeouts",
}
def delegate(*kinds: DeliverableKind) -> Call:
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
def approve(phase: PhaseId) -> Call:
return Call("request_approval", {"phase": phase.value, "summary": "done"})
def specialist(kind: DeliverableKind) -> str:
return DEPENDENCY_GRAPH[kind].specialist
def failing_validation() -> Call:
call = sample_submission(K.VALIDATION_RESULTS)
verdicts = json.loads(call.args["attachments"]["verdicts.json"])
verdicts["checks"][1] = {**FAILED, "verdict": "fail"}
verdicts["proposed_additions"] = [ADDITION]
return Call(
call.tool,
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
)
def turns_with_one_revision() -> dict:
once = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
return {
**once,
"orchestrator": [
# The first call is refused: requirements are not done yet.
[delegate(K.TECHNICAL_DECOMPOSITION), delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
[
approve(P.SOLUTION_ARCHITECTURE),
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
],
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[
Call(
"reopen",
{"kind": "deployment_guidance", "reason": "Fix the ingress block."},
),
delegate(K.DEPLOYMENT_GUIDANCE),
],
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[approve(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
[approve(P.PACKAGING), Call("finish")],
],
specialist(K.DEPLOYMENT_GUIDANCE): [
[
Call("submit_deliverable", {"content": "No headings yet."}),
sample_submission(K.DEPLOYMENT_GUIDANCE),
],
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
],
specialist(K.VALIDATION_RESULTS): [
[failing_validation()],
[sample_submission(K.VALIDATION_RESULTS)],
],
}
async def run(tmp_path, turns, **options):
recorder = InMemoryRecorder()
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=FakeAgentRuntime(turns),
roles=ROLES,
recorder=recorder,
checkpointer=InMemorySaver(),
output_root=tmp_path,
skill_versions={"solution-architecture": "abc123"},
**options,
)
result = await engine.start("Ingestion", pre_approved=True)
return result, recorder
def of_type(record: dict, kind: str) -> list[dict]:
return [c for c in record["candidates"] if c["type"] == kind]
async def test_a_finished_execution_records_what_went_wrong(tmp_path):
result, recorder = await run(tmp_path, turns_with_one_revision())
assert result.status == "completed"
record = recorder.lessons[result.execution_id]
assert of_type(record, "revision") == [
{
"type": "revision",
"deliverable": "deployment_guidance",
"reason": "Fix the ingress block.",
"requested_by": "orchestrator",
}
]
assert of_type(record, "failed_check") == [
{"type": "failed_check", "attempt": 1, **FAILED}
]
assert of_type(record, "proposed_addition") == [
{"type": "proposed_addition", "attempt": 1, **ADDITION}
]
refusals = of_type(record, "refusal")
assert {(r["role"], r["tool"], r["deliverable"]) for r in refusals} == {
("orchestrator", "delegate", None),
(
specialist(K.DEPLOYMENT_GUIDANCE),
"submit_deliverable",
"deployment_guidance",
),
}
assert all(r["message"].startswith(("Refused", "Not submitted")) for r in refusals)
async def test_the_record_names_the_execution_and_the_skills_loaded(tmp_path):
result, recorder = await run(tmp_path, turns_with_one_revision())
record = recorder.lessons[result.execution_id]
assert record["execution_id"] == result.execution_id
assert record["status"] == "completed"
assert record["role_skills"]["orchestrator"] == ["solution-architecture"]
assert record["skill_versions"] == {"solution-architecture": "abc123"}
async def test_the_record_is_also_written_beside_the_deliverables_tree(tmp_path):
result, recorder = await run(tmp_path, turns_with_one_revision())
path = (
tmp_path
/ "lessons"
/ "candidates"
/ "unassigned"
/ f"{result.execution_id}.json"
)
assert (
json.loads(path.read_text(encoding="utf-8"))
== (recorder.lessons[result.execution_id])
)
async def test_an_execution_that_ends_with_warnings_still_records_lessons(tmp_path):
turns = {
"orchestrator": [[delegate(K.REQUIREMENTS)], []],
specialist(K.REQUIREMENTS): [[]], # stops without submitting
}
result, recorder = await run(tmp_path, turns)
assert result.status == "completed_with_warnings"
record = recorder.lessons[result.execution_id]
assert of_type(record, "unfinished") == [
{
"type": "unfinished",
"deliverable": "requirements",
"role": specialist(K.REQUIREMENTS),
"reason": "stopped",
}
]
# Only the scope had been produced, so only the scope was written.
written = {path.name for path in tmp_path.rglob("deliverables/*.md")}
assert written == {"scope.md"}
def test_a_cloud_read_refusal_keeps_only_its_leading_words():
"""What a cloud-read tool says may describe the user's environment."""
item = refusal(
"requirements-analyst",
"requirements",
"compute__list_instances",
"cloud_read",
"Refused: project acme-prod-1234 does not have the API enabled.",
)
assert item["tool"] == "compute__list_instances"
assert item["message"] == "Refused"
assert "acme-prod-1234" not in item["message"]
def test_refusal_messages_are_bounded():
item = refusal(
"validator", "validation_results", "terraform_validate", "general", "x" * 900
)
assert len(item["message"]) == 500

View File

@@ -0,0 +1,105 @@
"""An Execution's files are on disk as soon as each Deliverable is submitted.
Before this they were written only when an Execution ended, so a process that was
killed, or an Execution left waiting for its user, had nothing to show.
"""
import json
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import FakeAgentRuntime
from execution_engine.adk.runtime import NoDecider, RoleSpec
from execution_engine.agent import ExecutionEngine
from execution_engine.baseline import BaselineError, load_baseline
from execution_engine.nodes.execution_nodes import _write_outputs
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from execution_engine.states.ledger import Ledger
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
WORKFLOW,
sample_submission,
)
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def folder(tmp_path, execution_id: str):
return tmp_path / "deliverables" / "cases" / "acme" / "executions" / execution_id
async def test_an_execution_waiting_for_its_user_has_written_what_exists(tmp_path):
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=FakeAgentRuntime(
{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
),
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
waiting = await engine.start("Ingest", customer_case="acme", execution_id="v1")
# It stopped for the first Approval: only the scope has been produced.
assert waiting.status == "awaiting_user"
out = folder(tmp_path, "v1")
manifest = json.loads((out / "execution.json").read_text())
assert manifest["status"] == "in_progress"
assert set(manifest["deliverables"]) == {"scope"}
assert (out / "deliverables" / "scope.md").read_text()
assert (out / "scope.json").is_file() and (out / "scope.yaml").is_file()
assert manifest["solution_guide"] is None
# Unfinished work is never something a Change request can start from.
try:
load_baseline(tmp_path, "acme", "v1")
except BaselineError as refused:
assert "did not complete (in_progress)" in str(refused)
else:
raise AssertionError("an in-progress Execution was accepted as a Baseline")
def test_a_file_an_earlier_write_left_is_removed_with_the_revision(tmp_path):
state = {
"execution_id": "v1",
"customer_case": "acme",
"workflow_request": "Ingest",
"deliverables": {
"deployment_guidance": {
"content": "guide",
"attachments": {
"terraform/main.tf": "one",
"terraform/queue.tf": "two",
},
}
},
}
out = folder(tmp_path, "v1")
_write_outputs(tmp_path, state, "in_progress", Ledger(WORKFLOW))
assert (out / "terraform" / "queue.tf").is_file()
(out / "notes.txt").write_text("a person's own file")
# The Revision folded the queue into main.tf.
state["deliverables"]["deployment_guidance"]["attachments"] = {
"terraform/main.tf": "one and two"
}
_write_outputs(tmp_path, state, "completed", Ledger(WORKFLOW))
assert not (out / "terraform" / "queue.tf").exists()
assert (out / "terraform" / "main.tf").read_text() == "one and two"
# Only files this Execution wrote are ever removed.
assert (out / "notes.txt").is_file()
assert json.loads((out / "execution.json").read_text())["status"] == "completed"

View File

@@ -0,0 +1,639 @@
"""The design advisor's fan-out to one advisor per architecture-framework pillar."""
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.runtime import RoleSpec, ToolResult, ToolSpec
from execution_engine.tools.toolkit import make_tool_factory
from tests.reference_workflow.hooks import PILLAR_ADVISOR
from tests.reference_workflow.pillars import CONSULT_TOOL, pillar_consultation_tool
from tests.reference_workflow.workflow import WORKFLOW, with_pillars
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
PILLARS = {
"security": "waf-security",
"reliability": "waf-reliability",
"sustainability": "waf-sustainability",
}
ADVISOR = RoleSpec(PILLAR_ADVISOR, "Advise on one pillar.", budget=5)
def advice(pillar: str) -> list[Call]:
return [Call("submit_advice", {"content": f"Use strong defaults for {pillar}."})]
def turn_for_pillar(task):
return advice(task.context["pillar"])
async def consult(tool: ToolSpec, **args):
return await tool.handler(**args)
async def test_consults_every_pillar_with_only_its_skill():
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
result = await consult(tool, brief="Event ingestion on serverless compute.")
assert result.outcome is None
for pillar in PILLARS:
assert f"## {pillar}" in result.message
assert f"Use strong defaults for {pillar}." in result.message
runs = {run.task.context["pillar"]: run for run in runtime.runs}
assert set(runs) == set(PILLARS)
assert (
runs["security"].task.context["brief"]
== "Event ingestion on serverless compute."
)
assert runtime.runs[0].tools == ("submit_advice",)
async def test_advisors_are_scoped_to_their_pillar_skill():
seen: dict[str, tuple[str, ...]] = {}
class RecordingRuntime(FakeAgentRuntime):
async def run(self, role, task, tools):
seen[task.context["pillar"]] = role.skills
return await super().run(role, task, tools)
runtime = RecordingRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
await consult(pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b")
assert seen == {pillar: (skill,) for pillar, skill in PILLARS.items()}
async def test_a_subset_of_pillars_can_be_consulted():
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
result = await consult(tool, brief="b", pillars=["reliability"])
assert "## reliability" in result.message
assert "## security" not in result.message
async def test_unknown_pillars_are_refused():
tool = pillar_consultation_tool(FakeAgentRuntime({}), ADVISOR, PILLARS)
result = await consult(tool, brief="b", pillars=["speed"])
assert result.message.startswith("Refused")
assert "security" in result.message
async def test_an_advisor_that_fails_is_reported_without_losing_the_others():
def maybe_fail(task):
if task.context["pillar"] == "reliability":
return [Call("submit_advice", {"content": ""})]
return advice(task.context["pillar"])
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [maybe_fail] * 3})
result = await consult(
pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b"
)
assert "Use strong defaults for security." in result.message
assert "## reliability\n\nNo guidance:" in result.message
async def test_tool_factory_assembles_local_mcp_and_consultation_tools():
async def noop(**_):
return ToolResult("ok")
def tool(name, tier="general"):
return ToolSpec(name, name, {"type": "object", "properties": {}}, noop, tier)
class Catalog:
def tools_for(self, server_ids):
available = {
"docs": [tool("docs__search")],
"compute": [tool("compute__list_instances", "cloud_read")],
}
return [t for sid in server_ids for t in available.get(sid, [])]
advisor = RoleSpec(PILLAR_ADVISOR, "", budget=5, mcp_servers=("docs", "compute"))
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]})
factory = make_tool_factory(
with_pillars({"security": "waf-security"}),
runtime,
{PILLAR_ADVISOR: advisor},
local_tools=[tool("mermaid_check"), tool("terraform_plan", "cloud_read")],
catalog=Catalog(),
)
diagrammer = RoleSpec("diagrammer", "", budget=5, tools=("mermaid_check",))
analyst = RoleSpec("requirements-analyst", "", budget=5, mcp_servers=("compute",))
assert [t.name for t in factory(diagrammer, "architecture_diagram", {})] == [
"mermaid_check"
]
assert [t.name for t in factory(analyst, "requirements", {})] == [
"compute__list_instances"
]
[consult_tool] = factory(
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
)
await consult_tool.handler(brief="b")
assert runtime.runs[0].tools == ("submit_advice", "docs__search")
async def test_tool_factory_gives_the_consultation_only_to_the_design_advisor():
roles = {PILLAR_ADVISOR: ADVISOR}
factory = make_tool_factory(with_pillars(PILLARS), FakeAgentRuntime({}), roles)
design = factory(
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
)
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition", {})
assert [tool.name for tool in design] == [CONSULT_TOOL]
assert list(other) == []
# --- advice remembered across Revisions (iteration 4, step 3) --------------------------
async def test_naming_pillars_reconsults_only_those_and_returns_the_rest_from_memory():
memory: dict = {}
first = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
await consult(
pillar_consultation_tool(first, ADVISOR, PILLARS, memory=memory), brief="v1"
)
def revised(task):
pillar = task.context["pillar"]
return [Call("submit_advice", {"content": f"Revised advice for {pillar}."})]
second = FakeAgentRuntime({PILLAR_ADVISOR: [revised]})
tool = pillar_consultation_tool(second, ADVISOR, PILLARS, memory=memory)
result = await consult(tool, brief="v2", pillars=["reliability"])
assert [run.task.context["pillar"] for run in second.runs] == ["reliability"]
assert "## reliability\n\nRevised advice for reliability." in result.message
assert (
"## security (from your earlier consultation)\n\n"
"Use strong defaults for security." in result.message
)
assert memory["pillar_advice"]["reliability"] == "Revised advice for reliability."
assert set(memory["pillar_advice"]) == set(PILLARS)
async def test_omitting_pillars_consults_every_pillar_afresh():
memory = {"pillar_advice": {pillar: "old" for pillar in PILLARS}}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
result = await consult(tool, brief="v2")
assert len(runtime.runs) == 3
assert "earlier consultation" not in result.message
assert "old" not in memory["pillar_advice"].values()
async def test_failed_advice_is_not_remembered():
memory: dict = {}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [advice("security"), []]})
tool = pillar_consultation_tool(
runtime, ADVISOR, {"security": "s", "reliability": "r"}, memory=memory
)
result = await consult(tool, brief="b")
assert "## reliability\n\nNo guidance" in result.message
assert set(memory["pillar_advice"]) == {"security"}
async def test_the_fan_out_shares_the_cap_and_a_cap_of_one_does_not_deadlock():
import asyncio
from execution_engine.adk.runtime import RunLimiter
limiter = RunLimiter(1)
in_flight = 0
peak = 0
class SlowRuntime(FakeAgentRuntime):
async def run(self, role, task, tools):
nonlocal in_flight, peak
in_flight += 1
peak = max(peak, in_flight)
await asyncio.sleep(0.01)
try:
return await super().run(role, task, tools)
finally:
in_flight -= 1
runtime = SlowRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, limiter=limiter)
async with limiter.slot(): # the design advisor's own run holds the only slot
result = await asyncio.wait_for(consult(tool, brief="b"), timeout=5)
assert peak == 1
assert result.message.count("Use strong defaults") == 3
async def test_pillar_advice_survives_a_revision_through_the_engine(tmp_path):
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.agent import ExecutionEngine
from execution_engine.recorder import InMemoryRecorder
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
DeliverableKind,
PhaseId,
sample_submission,
)
K, P = DeliverableKind, PhaseId
design = DEPENDENCY_GRAPH[K.DESIGN_RECOMMENDATIONS].specialist
roles = {
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
PILLAR_ADVISOR: ADVISOR,
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def delegate(*kinds):
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
def approve(phase):
return Call("request_approval", {"phase": phase.value, "summary": "done"})
submit = sample_submission(K.DESIGN_RECOMMENDATIONS)
runtime = FakeAgentRuntime(
{
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[
Call(
"reopen",
{"kind": "design_recommendations", "reason": "Tighten IAM."},
),
delegate(K.DESIGN_RECOMMENDATIONS),
],
[], # stops here: a pre-approved Execution ends with warnings
],
DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist: [
[sample_submission(K.REQUIREMENTS)]
],
DEPENDENCY_GRAPH[K.TECHNICAL_DECOMPOSITION].specialist: [
[sample_submission(K.TECHNICAL_DECOMPOSITION)]
],
DEPENDENCY_GRAPH[K.PRODUCT_SELECTION].specialist: [
[sample_submission(K.PRODUCT_SELECTION)]
],
design: [
[Call(CONSULT_TOOL, {"brief": "v1"}), submit],
[Call(CONSULT_TOOL, {"brief": "v2", "pillars": ["security"]}), submit],
],
PILLAR_ADVISOR: [turn_for_pillar] * 4,
}
)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
tool_factory=make_tool_factory(with_pillars(PILLARS), runtime, roles),
)
result = await engine.start("Ingestion", pre_approved=True)
advisor_runs = [r for r in runtime.runs if r.role == PILLAR_ADVISOR]
assert [r.task.context["pillar"] for r in advisor_runs[3:]] == ["security"]
second = [r for r in runtime.runs if r.role == design][1]
assert "## reliability (from your earlier consultation)" in second.results[0][1]
stored = result.state["working"]["design_recommendations"]["pillar_advice"]
assert set(stored) == set(PILLARS)
# --- each advisor reviews its pillar's checklist ----------------------------------------
def reviewed(task):
pillar = task.context["pillar"]
return [
Call(
"submit_advice",
{
"content": f"Use strong defaults for {pillar}.",
"checklist": [
{
"item": f"{pillar} item one | with a bar",
"status": "addressed",
"note": "Covered by the first recommendation.",
},
{
"item": f"{pillar} item two",
"status": "not_applicable",
"note": "An organisation's process, not a design decision.",
},
{"item": "malformed", "status": "maybe", "note": "dropped"},
],
},
)
]
async def test_each_advisors_checklist_review_is_kept_beside_its_advice():
from tests.reference_workflow.pillars import CHECKLIST_KEY
memory: dict = {}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
result = await consult(tool, brief="b")
# The design advisor is given the advice, not the review: it does not copy it.
assert "item one" not in result.message
assert set(memory[CHECKLIST_KEY]) == set(PILLARS)
review = memory[CHECKLIST_KEY]["security"]
assert [entry["status"] for entry in review] == ["addressed", "not_applicable"]
async def test_the_review_is_written_into_the_deliverable_by_code():
from execution_engine.adk.runtime import Submitted
from execution_engine.tools.execution import specialist_tools
from tests.reference_workflow.pillars import REVIEW_HEADING
from tests.reference_workflow.workflow import DeliverableKind, sample_submission
memory: dict = {}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
await consult(
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
)
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
(submit,) = (
tool
for tool in specialist_tools(
WORKFLOW, kind, (), allow_cloud_read=False, working=memory
)
if tool.name == "submit_deliverable"
)
call = sample_submission(kind)
first = await submit.handler(**call.args)
# A Revision submits content that already carries the section.
again = await submit.handler(
content=first.outcome.content, attachments=call.args["attachments"]
)
assert isinstance(first.outcome, Submitted)
content = first.outcome.content
assert f"## {REVIEW_HEADING}" in content and "### security" in content
assert (
"| security item one \\| with a bar | Addressed | Covered by the first "
"recommendation. |"
) in content
assert "| security item two | Does not apply |" in content
assert "malformed" not in content
assert again.outcome.content.count(f"## {REVIEW_HEADING}") == 1
async def test_with_no_review_nothing_is_added():
from execution_engine.tools.execution import specialist_tools
from tests.reference_workflow.pillars import REVIEW_HEADING
from tests.reference_workflow.workflow import DeliverableKind, sample_submission
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
(submit,) = (
tool
for tool in specialist_tools(
WORKFLOW, kind, (), allow_cloud_read=False, working={}
)
if tool.name == "submit_deliverable"
)
result = await submit.handler(**sample_submission(kind).args)
assert REVIEW_HEADING not in result.outcome.content
# --- only the pillars a change touches are consulted again -------------------------------
async def test_a_change_names_the_pillars_it_affects_directly():
from execution_engine.adk.fake_runtime import FakeDecider
from execution_engine.adk.runtime import Answer
from tests.reference_workflow.pillars import pillars_touched
decider = FakeDecider(
{
"security": Answer(1.9, 0.9),
"reliability": Answer(1.0, 0.6),
"sustainability": Answer(0.2, 0.8),
}
)
chosen = await pillars_touched(
decider, list(PILLARS), "Use customer-managed keys.", scope="e1"
)
state, questions, scope = decider.asked[0]
assert chosen == ["security"]
assert scope == "e1" and state.endswith("Use customer-managed keys.")
assert questions["security"].kind == "score"
assert questions["security"].options == (
"not at all",
"only indirectly",
"directly",
)
async def test_a_change_that_touches_no_pillar_is_an_answer_and_no_decision_is_not():
from execution_engine.adk.fake_runtime import FakeDecider
from execution_engine.adk.runtime import Answer, NoDecider
from tests.reference_workflow.pillars import pillars_touched
nothing = FakeDecider(dict.fromkeys(PILLARS, Answer(0.0, 0.99)))
assert await pillars_touched(nothing, list(PILLARS), "Rename a page.") == []
assert await pillars_touched(NoDecider(), list(PILLARS), "Rename a page.") is None
assert await pillars_touched(nothing, list(PILLARS), " ") is None
async def test_an_empty_list_of_pillars_consults_none_and_returns_what_was_said():
memory: dict = {}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
await consult(tool, brief="b")
again = await consult(tool, brief="b", pillars=[])
assert len(runtime.runs) == 3 # nobody was consulted a second time
for pillar in PILLARS:
assert f"## {pillar} (from your earlier consultation)" in again.message
# --- what the request does not say is a question for the customer ------------------------
def unsure(question: str | None = "Do you have a central identity provider?"):
def turn(task):
entry = {
"item": "Identity is managed centrally",
"status": "needs_customer_input",
"note": "The request does not say how staff identities are managed.",
}
if question is not None:
entry["question"] = question
first = Call("submit_advice", {"content": "Advice.", "checklist": [entry]})
# A second attempt, used only when the first is refused.
fixed = Call(
"submit_advice",
{
"content": "Advice.",
"checklist": [{**entry, "question": "Who manages staff identities?"}],
},
)
return [first, fixed]
return turn
async def test_an_item_the_request_cannot_settle_is_kept_with_its_question():
from tests.reference_workflow.pillars import CHECKLIST_KEY, customer_questions
memory: dict = {}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [unsure()] * 3})
await consult(
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
)
(entry,) = memory[CHECKLIST_KEY]["security"]
assert entry["status"] == "needs_customer_input"
assert customer_questions(memory)[0] == {
"pillar": "security",
"item": "Identity is managed centrally",
"question": "Do you have a central identity provider?",
}
assert len(customer_questions(memory)) == len(PILLARS)
async def test_an_item_for_the_customer_must_carry_the_question_to_ask():
from tests.reference_workflow.pillars import CHECKLIST_KEY
memory: dict = {}
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [unsure(question=None)] * 3})
await consult(
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
)
refused, accepted = (reply for _, reply in runtime.runs[0].results)
assert refused.startswith("Not submitted: an item to confirm with the customer")
assert "Identity is managed centrally" in refused
(entry,) = memory[CHECKLIST_KEY]["security"]
assert entry["question"] == "Who manages staff identities?"
assert accepted == "Advice submitted."
def test_the_questions_are_reported_as_missing_information_not_as_faults():
from tests.reference_workflow.pillars import (
CHECKLIST_KEY,
QUESTIONS_HEADING,
render_checklist_review,
)
working = {
CHECKLIST_KEY: {
"security": [
{
"item": "Scans run in the pipeline",
"status": "addressed",
"note": "DR-3",
},
{
"item": "Identity is managed centrally",
"status": "needs_customer_input",
"note": "Not said.",
"question": "Do you have a central identity provider?",
},
]
}
}
text = render_checklist_review(working)
assert "| Identity is managed centrally | To confirm with the customer |" in text
assert f"### {QUESTIONS_HEADING}" in text
assert "They are not faults in the design" in text
assert (
"| security | Identity is managed centrally | Do you have a central "
"identity provider? |"
) in text
# With nothing to ask there is no questions section.
only_addressed = {
CHECKLIST_KEY: {"security": working[CHECKLIST_KEY]["security"][:1]}
}
assert QUESTIONS_HEADING not in render_checklist_review(only_addressed)
async def test_an_execution_writes_its_questions_for_the_customer(tmp_path):
import json
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.runtime import NoDecider
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from tests.reference_workflow.pillars import CONSULT_TOOL
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
DeliverableKind,
sample_submission,
)
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
advisor = DEPENDENCY_GRAPH[kind].specialist
roles = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
PILLAR_ADVISOR: ADVISOR,
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
turns = {
spec.specialist: [[sample_submission(k)]]
for k, spec in DEPENDENCY_GRAPH.items()
}
turns[advisor] = [[Call(CONSULT_TOOL, {"brief": "b"}), sample_submission(kind)]]
turns[PILLAR_ADVISOR] = [unsure()] * 3
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
tool_factory=make_tool_factory(with_pillars(PILLARS), runtime, roles),
)
result = await engine.start("Ingest", customer_case="acme", pre_approved=True)
assert result.status == "completed"
out = next(tmp_path.rglob("execution.json")).parent
manifest = json.loads((out / "execution.json").read_text())
assert manifest["customer_questions"] == "customer-questions.md"
questions = (out / "customer-questions.md").read_text()
assert questions.startswith("# Questions for the customer")
assert "Do you have a central identity provider?" in questions
assert (
"### Questions for the customer"
in (out / "deliverables" / "design_recommendations.md").read_text()
)

View File

@@ -0,0 +1,100 @@
"""Prompt text for plain clients and the /generate response mapping."""
import json
from execution_engine.agent import Interaction
from tests.reference_workflow.presentation import describe_prompt, to_generate_response
def test_approval_prompt_explains_the_structured_reply():
text = describe_prompt(
{
"approval": {
"phase": "solution_architecture",
"summary": "Cloud Run and Pub/Sub",
"deliverables": ["product_selection", "target_architecture"],
}
}
)
reply = {"approval": {"phase": "solution_architecture", "decision": "approve"}}
assert json.dumps(reply) in text
assert "never approves" in text
def test_questions_are_numbered_with_their_source():
text = describe_prompt(
{
"questions": [
{"from": "requirements", "question": "RTO?"},
{"from": "orchestrator", "question": "Region?"},
]
}
)
assert text.startswith("1. (requirements) RTO?\n2. (orchestrator) Region?")
def test_generate_response_maps_deliverables_onto_legacy_artifacts():
requirements = (
"# Requirements\n## 2.3. Current state\nThree VMs.\n### Detail\nNginx.\n"
"## Dependencies\nNone.\n"
)
interaction = Interaction(
"exec-1",
"completed_with_warnings",
None,
{
"session_id": "s-1",
"warnings": ["revision limit"],
"deliverables": {
"requirements": {"content": requirements, "attachments": {}},
"deployment_guidance": {
"content": "# Guide",
"attachments": {"terraform/main.tf": "resource {}"},
},
},
"ledger": {
"deliverables": {"requirements": {"revisions": 2}},
"phases": {
"requirements_discovery": "approved",
"solution_architecture": "open",
},
},
},
)
body = to_generate_response(interaction, {})
assert body["status"] == "completed_with_warnings"
assert body["validation_passed"] is False
assert body["total_loop_iterations"] == 3
assert body["current_phase"] == "solution_architecture"
assert body["artifacts"]["source_discovery_doc"] == "Three VMs.\n### Detail\nNginx."
assert body["artifacts"]["terraform_code"] == "resource {}"
assert body["artifacts"]["solution_guide"] is None
assert body["warnings"] == ["revision limit"]
def test_the_design_approval_says_what_the_user_is_committing_to():
text = describe_prompt(
{
"approval": {
"phase": "solution_architecture",
"summary": "Cloud Run and Pub/Sub",
"deliverables": ["target_architecture"],
"target_architecture": {
"elements": 9,
"relationships": 11,
"checks": 23,
"not_applicable": [
{"recommendation": "DR-9", "reason": "Single region."}
],
},
}
}
)
assert "9 Elements, 11 Relationships and 23 Checks" in text
assert "No infrastructure code is written until you approve it" in text
assert "Recommendation DR-9 is treated as not applicable: Single region." in text

View File

@@ -0,0 +1,89 @@
"""The database recorder and the configured checkpointer."""
import sqlite3
import pytest
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from execution_engine.checkpointing import open_checkpointer
from execution_engine.config import get_settings
from execution_engine.database import DatabaseManager
from execution_engine.recorder import DatabaseRecorder
from execution_engine.states.ledger import LedgerEvent
@pytest.fixture
def sqlite_db(tmp_path, monkeypatch):
path = tmp_path / "records.db"
monkeypatch.setattr(get_settings(), "SQLITE_FALLBACK_DB", str(path))
return DatabaseManager(db_url="sqlite:///unused"), path
async def test_database_recorder_writes_events_submissions_and_status(sqlite_db):
manager, path = sqlite_db
recorder = DatabaseRecorder(manager)
await recorder.record_events(
"exec-1",
[
LedgerEvent(
1, "submitted", "requirements", "requirements-analyst", "attempt 1"
)
],
)
await recorder.record_submission(
"exec-1", "deployment_guidance", "# Guide", {"terraform/main.tf": "resource {}"}
)
await recorder.record_status(
"exec-1",
session_id="session-1",
workflow_request="Ingestion",
status="awaiting_user",
current_phase="solution_architecture",
details={"revisions": 2},
)
with sqlite3.connect(path) as conn:
logs = conn.execute(
"SELECT iteration, review_status, reviewer_agent, feedback "
"FROM orchestrator_review_logs WHERE execution_id = 'exec-1'"
).fetchall()
artifacts = conn.execute(
"SELECT artifact_type, file_path FROM adk_artifacts WHERE execution_id = 'exec-1' "
"ORDER BY file_path"
).fetchall()
execution = conn.execute(
"SELECT status, current_phase, loop_count FROM workflow_executions "
"WHERE execution_id = 'exec-1'"
).fetchone()
session = conn.execute(
"SELECT agent_name, state_data FROM adk_sessions WHERE session_id = 'session-1'"
).fetchone()
assert logs == [(1, "submitted", "requirements-analyst", "requirements: attempt 1")]
assert artifacts == [
("deployment_guidance", "deliverables/deployment_guidance.md"),
("deployment_guidance", "terraform/main.tf"),
]
assert execution == ("awaiting_user", "solution_architecture", 2)
assert session[0] == "solution-architect"
assert "exec-1" in session[1]
async def test_checkpointer_follows_settings(tmp_path, monkeypatch):
settings = get_settings()
monkeypatch.setattr(settings, "CHECKPOINTER", "memory")
async with open_checkpointer(settings) as saver:
assert isinstance(saver, InMemorySaver)
monkeypatch.setattr(settings, "CHECKPOINTER", "sqlite")
monkeypatch.setattr(settings, "CHECKPOINT_DB_PATH", str(tmp_path / "cp.db"))
async with open_checkpointer(settings) as saver:
assert isinstance(saver, AsyncSqliteSaver)
monkeypatch.setattr(settings, "CHECKPOINTER", "redis")
with pytest.raises(ValueError, match="redis"):
async with open_checkpointer(settings):
pass

View File

@@ -0,0 +1,888 @@
"""The Requirements chain: the Target architecture graph and the data files around it.
ADR-0005: the outcome is understood before code is written, and validation answers
the graph's Checks and nothing else.
"""
import json
import pytest
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.runtime import RoleSpec
from execution_engine.agent import ExecutionEngine
from execution_engine.recorder import InMemoryRecorder
from execution_engine.tools.execution import specialist_tools
from tests.reference_workflow.chain import unimplementable_checks, validation_outcome
from tests.reference_workflow.local import mermaid_check
from tests.reference_workflow.target_architecture import (
GraphError,
chain_gaps,
parse_graph,
render_mermaid,
)
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
SAMPLE_DATA,
WORKFLOW,
DeliverableKind,
PhaseId,
check_submission,
sample_submission,
)
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
K = DeliverableKind
P = PhaseId
REQUIREMENTS = json.loads(SAMPLE_DATA["requirements.json"])
COMPONENTS = json.loads(SAMPLE_DATA["components.json"])
PRODUCTS = json.loads(SAMPLE_DATA["products.json"])
RECOMMENDATIONS = json.loads(SAMPLE_DATA["recommendations.json"])
def graph_data() -> dict:
return json.loads(SAMPLE_DATA["target-architecture.json"])
def problems(data: dict) -> tuple[str, ...]:
with pytest.raises(GraphError) as raised:
parse_graph(json.dumps(data))
return raised.value.problems
def gaps(data: dict, **overrides) -> tuple[str, ...]:
links = {
"requirements": REQUIREMENTS,
"components": COMPONENTS,
"products": PRODUCTS,
"recommendations": RECOMMENDATIONS,
**overrides,
}
return chain_gaps(parse_graph(json.dumps(data)), **links)
def upstream(*kinds: DeliverableKind, **replaced: str) -> dict:
"""The Deliverables a submission builds on, from the coherent sample."""
built = {}
for kind in kinds:
args = sample_submission(kind).args
attachments = {
name: replaced.get(name, text) for name, text in args["attachments"].items()
}
built[kind.value] = {"content": args["content"], "attachments": attachments}
return built
DESIGN = (
K.REQUIREMENTS,
K.TECHNICAL_DECOMPOSITION,
K.PRODUCT_SELECTION,
K.DESIGN_RECOMMENDATIONS,
)
# --- the graph is well-formed ----------------------------------------------------------
def test_the_sample_graph_parses_into_elements_relationships_and_checks():
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
assert [element.id for element in graph.elements] == ["E-1", "E-2"]
assert graph.elements[1].component is None # a supporting Element
assert [(r.source, r.target) for r in graph.relationships] == [("E-1", "E-2")]
assert graph.check_ids == ("K-1", "K-2")
assert graph.summary() == {
"elements": 2,
"relationships": 1,
"checks": 2,
"not_applicable": [],
}
def test_a_relationship_must_join_two_elements():
data = graph_data()
data["relationships"][0]["to"] = "E-9"
assert problems(data) == (
"target-architecture.json: Relationship R-1 to E-9, which is not an Element",
)
def test_a_check_must_apply_to_something_in_the_graph():
data = graph_data()
data["checks"][0]["applies_to"] = "E-9"
assert problems(data) == (
(
"target-architecture.json: Check K-1 applies to E-9, "
"which is not an Element or Relationship"
),
"target-architecture.json: Element E-1 has no Check",
)
def test_every_element_needs_a_check_and_every_id_is_used_once():
data = graph_data()
data["checks"] = [data["checks"][0]]
data["relationships"][0]["id"] = "E-1"
assert problems(data) == (
"target-architecture.json: id E-1 is used more than once",
"target-architecture.json: Element E-2 has no Check",
)
def test_a_check_without_a_statement_is_named():
data = graph_data()
data["checks"][1]["statement"] = " "
assert (
"target-architecture.json.checks[1]: needs a non-empty statement"
in problems(data)
)
@pytest.mark.parametrize("text", ["not json", "[]", '{"elements": "many"}'])
def test_a_graph_that_is_not_the_right_shape_is_refused(text):
with pytest.raises(GraphError):
parse_graph(text)
# --- the Requirements chain has no gap -------------------------------------------------
def test_the_sample_chain_is_complete():
assert gaps(graph_data()) == ()
def test_a_non_functional_requirement_with_no_check_is_a_gap():
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
assert gaps(graph_data(), requirements=[*REQUIREMENTS, retention]) == (
"Requirement NFR-2 has no Check",
)
def test_a_functional_requirement_is_carried_by_its_components_element():
# FR-1 is the source of no Check, but C-1 serves it and E-1 realises C-1.
assert not [gap for gap in gaps(graph_data()) if "FR-1" in gap]
unserved = [{**COMPONENTS[0], "requirements": ["NFR-1"]}]
assert gaps(graph_data(), components=unserved) == ("Requirement FR-1 has no Check",)
def test_a_component_with_no_element_is_a_gap():
replay = {"id": "C-2", "name": "Replay service", "requirements": []}
assert gaps(graph_data(), components=[*COMPONENTS, replay]) == (
"Component C-2 has no Element",
)
def test_a_selected_product_that_no_element_uses_is_a_gap():
armour = [{"component": "C-1", "product": "Example Armor"}]
assert gaps(graph_data(), products=armour) == (
"Product Example Armor selected for C-1 appears on no Element",
)
def test_a_recommendation_needs_a_check_or_a_reason_it_does_not_apply():
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
recommendations = [*RECOMMENDATIONS, cmek]
assert gaps(graph_data(), recommendations=recommendations) == (
"Recommendation DR-2 has no Check",
)
excused = graph_data()
excused["not_applicable"] = [
{"recommendation": "DR-2", "reason": "No regulated data."}
]
assert gaps(excused, recommendations=recommendations) == ()
assert parse_graph(json.dumps(excused)).summary()["not_applicable"] == [
{"recommendation": "DR-2", "reason": "No regulated data."}
]
def test_a_check_must_come_from_a_requirement_or_recommendation():
data = graph_data()
data["checks"][0]["source"] = "my own idea"
assert gaps(data) == (
(
"Check K-1 has source my own idea, which is not a requirement or design "
"recommendation"
),
"Requirement NFR-1 has no Check",
)
def test_a_target_architecture_with_a_gap_cannot_be_submitted():
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
built_on = upstream(
*DESIGN, **{"requirements.json": json.dumps([*REQUIREMENTS, retention])}
)
args = sample_submission(K.TARGET_ARCHITECTURE).args
assert check_submission(
K.TARGET_ARCHITECTURE, args["content"], args["attachments"], built_on
) == ("Requirement NFR-2 has no Check",)
assert (
check_submission(
K.TARGET_ARCHITECTURE,
args["content"],
args["attachments"],
upstream(*DESIGN),
)
== ()
)
# --- the links before the graph --------------------------------------------------------
def test_requirements_need_ids_and_a_kind():
def missing(items) -> tuple[str, ...]:
args = sample_submission(K.REQUIREMENTS).args
return check_submission(
K.REQUIREMENTS, args["content"], {"requirements.json": json.dumps(items)}
)
assert missing(REQUIREMENTS) == ()
assert missing([]) == ("requirements.json: lists no requirements",)
assert missing([REQUIREMENTS[0], {**REQUIREMENTS[0], "kind": "must"}]) == (
"requirements.json: id FR-1 is used more than once",
"requirements.json: FR-1 kind must be one of functional, non_functional",
)
assert missing([{"id": "FR-1", "kind": "functional"}]) == (
"requirements.json[0]: needs text",
)
def test_every_functional_requirement_is_served_by_a_component():
args = sample_submission(K.TECHNICAL_DECOMPOSITION).args
def missing(components) -> tuple[str, ...]:
return check_submission(
K.TECHNICAL_DECOMPOSITION,
args["content"],
{"components.json": json.dumps(components)},
upstream(K.REQUIREMENTS),
)
assert missing(COMPONENTS) == ()
assert missing([{"id": "C-1", "name": "Ingress", "requirements": ["NFR-9"]}]) == (
"components.json: C-1 serves NFR-9, which is not a requirement",
"components.json: functional requirement FR-1 is served by no component",
)
def test_every_component_gets_a_product_and_recommendations_name_real_components():
selection = sample_submission(K.PRODUCT_SELECTION).args
advice = sample_submission(K.DESIGN_RECOMMENDATIONS).args
built_on = upstream(K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
assert check_submission(
K.PRODUCT_SELECTION,
selection["content"],
{"products.json": json.dumps([{"component": "C-7", "product": "X"}])},
built_on,
) == (
"products.json: C-7 is not a component",
"products.json: component C-1 has no product selected",
)
wandering = [{**RECOMMENDATIONS[0], "components": ["C-7"]}]
assert check_submission(
K.DESIGN_RECOMMENDATIONS,
advice["content"],
{"recommendations.json": json.dumps(wandering)},
built_on,
) == ("recommendations.json: DR-1 names C-7, which is not a component",)
# --- code and validation answer the graph ----------------------------------------------
EVERYTHING_BEFORE_CODE = (*DESIGN, K.TARGET_ARCHITECTURE)
def test_deployment_guidance_must_account_for_every_check():
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
partial = json.dumps(
[
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
{"check": "K-7", "file": "terraform/elsewhere.tf", "address": "b"},
]
)
missing = check_submission(
K.DEPLOYMENT_GUIDANCE,
args["content"],
{**args["attachments"], "implementation.json": partial},
upstream(*EVERYTHING_BEFORE_CODE),
)
assert missing == (
"implementation.json: K-7 is not a Check in the Target architecture",
"implementation.json: Check K-2 is not accounted for",
"implementation.json: K-7 names terraform/elsewhere.tf, which is not attached",
)
def test_a_check_that_cannot_be_implemented_is_accounted_for_with_its_reason():
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
implementation = json.dumps(
[
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
{
"check": "K-2",
"unimplementable": "example_service_account has no such argument",
},
]
)
missing = check_submission(
K.DEPLOYMENT_GUIDANCE,
args["content"],
{**args["attachments"], "implementation.json": implementation},
upstream(*EVERYTHING_BEFORE_CODE),
)
assert missing == ()
assert unimplementable_checks(implementation) == [
{"check": "K-2", "reason": "example_service_account has no such argument"}
]
assert unimplementable_checks(SAMPLE_DATA["implementation.json"]) == []
def test_an_unimplementable_entry_needs_its_reason():
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
implementation = json.dumps(
[
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
{"check": "K-2", "unimplementable": ""},
]
)
missing = check_submission(
K.DEPLOYMENT_GUIDANCE,
args["content"],
{**args["attachments"], "implementation.json": implementation},
upstream(*EVERYTHING_BEFORE_CODE),
)
assert missing == (
"implementation.json[0]: needs unimplementable",
"implementation.json: Check K-2 is not accounted for",
)
def verdicts(**changes) -> str:
data = json.loads(SAMPLE_DATA["verdicts.json"])
data.update(changes)
return json.dumps(data)
def validation_missing(text: str) -> tuple[str, ...]:
args = sample_submission(K.VALIDATION_RESULTS).args
return check_submission(
K.VALIDATION_RESULTS,
args["content"],
{"verdicts.json": text},
upstream(*EVERYTHING_BEFORE_CODE, K.DEPLOYMENT_GUIDANCE),
)
def test_validation_gives_exactly_one_verdict_per_check():
one = {"check": "K-1", "verdict": "pass", "evidence": "seen"}
invented = {"check": "K-9", "verdict": "fail", "evidence": "No Cloud Armor rule."}
assert validation_missing(SAMPLE_DATA["verdicts.json"]) == ()
assert validation_missing(verdicts(checks=[one, one, invented])) == (
"verdicts.json.checks: check K-1 is used more than once",
"verdicts.json.checks: K-9 is not a Check in the Target architecture",
"verdicts.json.checks: Check K-2 is not accounted for",
)
def test_the_validator_cannot_fail_the_code_on_something_the_graph_lacks():
"""What it notices beyond the Checks is a proposed addition, never a failure."""
addition = {
"statement": "Cloud Armor rate limiting is configured.",
"applies_to": "E-1",
"reason": "NFR-1",
}
text = verdicts(proposed_additions=[addition])
assert validation_missing(text) == ()
outcome = validation_outcome(text)
assert outcome["failed"] == []
assert outcome["proposed_additions"] == [addition]
def test_a_verdict_needs_evidence_and_a_known_value_and_the_dry_run_a_result():
vague = [
{"check": "K-1", "verdict": "probably", "evidence": "x"},
{"check": "K-2", "verdict": "pass", "evidence": " "},
]
assert validation_missing(verdicts(dry_run={"verdict": "ok"}, checks=vague)) == (
"verdicts.json.dry_run: needs a verdict (pass, fail, not_run) and evidence",
"verdicts.json.checks[1]: needs evidence",
"verdicts.json.checks: K-1 verdict must be one of pass, fail, not_verifiable",
"verdicts.json.checks: Check K-2 is not accounted for",
)
def test_a_validation_outcome_separates_faults_in_code_from_gaps_in_the_graph():
checks = [
{"check": "K-1", "verdict": "fail", "evidence": "min instances is 1"},
{"check": "K-2", "verdict": "not_verifiable", "evidence": "No such argument."},
]
outcome = validation_outcome(verdicts(checks=checks))
assert outcome["failed"] == [{"check": "K-1", "evidence": "min instances is 1"}]
assert outcome["not_verifiable"] == [
{"check": "K-2", "evidence": "No such argument."}
]
assert outcome["passed"] == []
assert outcome["dry_run"]["verdict"] == "pass"
# --- the diagram is a view of the graph ------------------------------------------------
async def test_the_diagram_shows_every_element_and_relationship_once():
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
diagram = render_mermaid(graph)
assert diagram == render_mermaid(graph) # deterministic
assert diagram.count("Ingress service") == 1
assert diagram.count("Ingress identity") == 1
assert diagram.count("-->") == 1
assert "runs as" in diagram
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
async def test_names_with_characters_mermaid_reserves_still_render():
data = graph_data()
data["elements"][0]["name"] = 'Ingest "edge" [v2] (public) | {x}'
data["elements"][0]["id"] = "svc/ingest.v2"
data["relationships"][0]["from"] = "svc/ingest.v2"
data["checks"][0]["applies_to"] = "svc/ingest.v2"
data["relationships"][0]["kind"] = "calls (HTTPS) | signed"
diagram = render_mermaid(parse_graph(json.dumps(data)))
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
assert "svc/ingest.v2" not in diagram
async def test_the_engine_draws_the_diagram_when_the_target_architecture_is_submitted():
submit = specialist_tools(
WORKFLOW,
K.TARGET_ARCHITECTURE,
extra=[],
allow_cloud_read=False,
upstream=upstream(*DESIGN),
)[0]
args = sample_submission(K.TARGET_ARCHITECTURE).args
result = await submit.handler(**args)
assert result.outcome.attachments["architecture.mmd"] == render_mermaid(
parse_graph(SAMPLE_DATA["target-architecture.json"])
)
async def test_a_gap_is_refused_to_the_specialist_with_the_missing_link_named():
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
built_on = upstream(
*DESIGN, **{"recommendations.json": json.dumps([*RECOMMENDATIONS, cmek])}
)
submit = specialist_tools(
WORKFLOW,
K.TARGET_ARCHITECTURE,
extra=[],
allow_cloud_read=False,
upstream=built_on,
)[0]
result = await submit.handler(**sample_submission(K.TARGET_ARCHITECTURE).args)
assert result.outcome is None
assert result.message == (
"Not submitted. The Deliverable is missing: Recommendation DR-2 has no Check"
)
# --- through the engine ----------------------------------------------------------------
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def delegate(*kinds: DeliverableKind) -> Call:
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
def approve(phase: PhaseId) -> Call:
return Call("request_approval", {"phase": phase.value, "summary": "done"})
def specialist(kind: DeliverableKind) -> str:
return DEPENDENCY_GRAPH[kind].specialist
def to_design_approval() -> list[list[Call]]:
return [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
]
def engine_for(tmp_path, turns: dict) -> tuple[ExecutionEngine, FakeAgentRuntime]:
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
)
return engine, runtime
async def test_the_user_approves_the_graph_before_any_code_is_written(tmp_path):
once = {
specialist(kind): [[sample_submission(kind)]]
for kind in (*DESIGN, K.TARGET_ARCHITECTURE)
}
engine, runtime = engine_for(
tmp_path,
{
**once,
"orchestrator": [
[delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY)],
[delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
# Tries to start the code before asking: refused, then asks.
[delegate(K.DEPLOYMENT_GUIDANCE), approve(P.SOLUTION_ARCHITECTURE)],
],
},
)
first = await engine.start("Ingest webhooks")
paused = await engine.resume(
first.execution_id,
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
)
assert paused.prompt["approval"]["phase"] == "solution_architecture"
assert paused.prompt["approval"]["target_architecture"] == {
"elements": 2,
"relationships": 1,
"checks": 2,
"not_applicable": [],
}
refused = runtime.runs[-1].results[0]
assert refused[0] == "delegate" and refused[1].startswith("Refused")
assert specialist(K.DEPLOYMENT_GUIDANCE) not in [run.role for run in runtime.runs]
async def test_the_orchestrator_is_told_what_validation_decided(tmp_path):
failing = json.loads(SAMPLE_DATA["verdicts.json"])
failing["checks"][0] = {
"check": "K-1",
"verdict": "fail",
"evidence": "min instances is 1",
}
failed_validation = sample_submission(K.VALIDATION_RESULTS)
failed_validation = Call(
failed_validation.tool,
{
**failed_validation.args,
"attachments": {"verdicts.json": json.dumps(failing)},
},
)
once = {
specialist(kind): [[sample_submission(kind)]]
for kind in (*DESIGN, K.TARGET_ARCHITECTURE, K.ARCHITECTURE_DESCRIPTION)
}
engine, runtime = engine_for(
tmp_path,
{
**once,
"orchestrator": [
*to_design_approval(),
[
approve(P.SOLUTION_ARCHITECTURE),
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
],
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[
Call(
"reopen",
{
"kind": "deployment_guidance",
"reason": "K-1 failed: min instances is 1",
},
),
delegate(K.DEPLOYMENT_GUIDANCE),
],
[],
],
specialist(K.DEPLOYMENT_GUIDANCE): [
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
],
specialist(K.VALIDATION_RESULTS): [[failed_validation]],
},
)
result = await engine.start("Ingest webhooks", pre_approved=True)
turns = [run for run in runtime.runs if run.role == "orchestrator"]
before, after = turns[6].task.context, turns[7].task.context
assert before["validation"] is None
assert after["validation"]["failed"] == [
{"check": "K-1", "evidence": "min instances is 1"}
]
assert after["validation"]["proposed_additions"] == []
# Once the code is reopened those verdicts are about work that no longer exists,
# so the next turn must not see them and reopen the same thing again.
assert turns[8].task.context["validation"] is None
# Revising the code leaves the approved design standing.
phases = result.state["ledger"]["phases"]
assert phases["solution_architecture"] == "approved"
revised = [
run for run in runtime.runs if run.role == specialist(K.DEPLOYMENT_GUIDANCE)
][1]
assert revised.task.context["feedback"] == ["K-1 failed: min instances is 1"]
# --- constraints list their conditions --------------------------------------------------
def test_a_non_functional_requirement_must_list_its_conditions():
requirements = [
{"id": "FR-1", "kind": "functional", "text": "Accept events."},
{"id": "NFR-1", "kind": "non_functional", "text": "Encrypted and retained."},
{
"id": "NFR-2",
"kind": "non_functional",
"text": "Available.",
"conditions": [],
},
]
args = sample_submission(K.REQUIREMENTS).args
missing = check_submission(
K.REQUIREMENTS,
args["content"],
{"requirements.json": json.dumps(requirements)},
)
needs = "needs `conditions`:"
assert [line.split(" a list")[0] for line in missing] == [
f"requirements.json: NFR-1 {needs}",
f"requirements.json: NFR-2 {needs}",
]
def test_a_recommendation_must_list_its_conditions():
recommendations = json.loads(SAMPLE_DATA["recommendations.json"])
del recommendations[0]["conditions"]
args = sample_submission(K.DESIGN_RECOMMENDATIONS).args
missing = check_submission(
K.DESIGN_RECOMMENDATIONS,
args["content"],
{"recommendations.json": json.dumps(recommendations)},
)
needs = "needs `conditions`:"
assert [line.split(" a list")[0] for line in missing] == [
f"recommendations.json: DR-1 {needs}"
]
def test_what_code_cannot_show_is_listed_apart_and_needs_no_check():
"""An outcome measured in operation is recorded, and no Check is asked for it."""
requirements = json.loads(SAMPLE_DATA["requirements.json"])
requirements.append(
{
"id": "NFR-2",
"kind": "non_functional",
"text": "Pages load within one second at p95.",
"conditions": [],
"outside_code": ["Measured p95 page load is under one second."],
}
)
args = sample_submission(K.REQUIREMENTS).args
accepted = check_submission(
K.REQUIREMENTS,
args["content"],
{"requirements.json": json.dumps(requirements)},
)
gaps = chain_gaps(
parse_graph(SAMPLE_DATA["target-architecture.json"]),
requirements=requirements,
components=json.loads(SAMPLE_DATA["components.json"]),
products=json.loads(SAMPLE_DATA["products.json"]),
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
)
assert accepted == ()
assert gaps == ()
def test_a_constraint_with_conditions_still_needs_a_check():
requirements = json.loads(SAMPLE_DATA["requirements.json"])
requirements.append(
{
"id": "NFR-2",
"kind": "non_functional",
"text": "Two zones, 99.9%.",
"conditions": ["The service runs in two zones."],
"outside_code": ["Measured availability is 99.9%."],
}
)
gaps = chain_gaps(
parse_graph(SAMPLE_DATA["target-architecture.json"]),
requirements=requirements,
components=json.loads(SAMPLE_DATA["components.json"]),
products=json.loads(SAMPLE_DATA["products.json"]),
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
)
assert gaps == ("Requirement NFR-2 has no Check",)
# --- what the customer stated reaches the requirements (ADR-0008) ------------------------
def scoped_upstream(**scope) -> dict:
import json
return {"scope": {"attachments": {"scope.json": json.dumps(scope)}}}
def requirement_problems(requirements: list[dict], **scope) -> list[str]:
import json
from tests.reference_workflow.chain import check_requirements
return check_requirements(
{"requirements.json": json.dumps(requirements)}, scoped_upstream(**scope)
)
STATED = {
"stated": {
"intent": {
"requirements": [{"id": "REQ-01", "statement": "Ready in 10 weeks."}]
},
"networking": {"topology": "hub_and_spoke"},
},
"binding": ["networking.topology"],
}
FUNCTIONAL = {"id": "FR-1", "kind": "functional", "text": "Accept events."}
CARRIED = {
"id": "NFR-1",
"kind": "non_functional",
"text": "The network is hub and spoke, ready in ten weeks.",
"conditions": ["A hub network is connected to each spoke network."],
"from": ["REQ-01", "networking.topology"],
}
def test_requirements_that_carry_everything_the_customer_stated_are_accepted():
assert requirement_problems([FUNCTIONAL, CARRIED], **STATED) == []
def test_a_customers_own_requirement_must_be_covered_by_name():
uncited = {**CARRIED, "from": ["networking.topology"]}
(problem,) = requirement_problems([FUNCTIONAL, uncited], **STATED)
assert "the customer's requirement REQ-01 is covered by no requirement" in problem
def test_a_stated_choice_must_be_carried_by_a_requirement():
"""Case 08: a stated topology was named in the design and not built, and every
Check passed, because nothing asked for it."""
uncarried = {**CARRIED, "from": ["REQ-01"]}
(problem,) = requirement_problems([FUNCTIONAL, uncarried], **STATED)
assert "the customer stated networking.topology and no requirement carries it" in (
problem
)
def test_a_stated_choice_is_carried_as_a_constraint_the_design_is_checked_against():
as_functional = {**CARRIED, "kind": "functional", "conditions": []}
outside_code_only = {
**CARRIED,
"conditions": [],
"outside_code": ["The topology is reviewed by the network team."],
}
for weak in (as_functional, outside_code_only):
problems = requirement_problems([FUNCTIONAL, weak], **STATED)
assert any(
"must be non_functional with at least one entry in `conditions`" in p
for p in problems
), weak
def test_with_no_scope_or_nothing_binding_requirements_are_as_before():
from tests.reference_workflow.chain import check_requirements
plain = [FUNCTIONAL]
import json
assert check_requirements({"requirements.json": json.dumps(plain)}, None) == []
assert requirement_problems(plain, stated={"intent": {"statement": "x"}}) == []
def test_what_was_assumed_for_the_customer_does_not_bind_the_design():
from execution_engine.scope import Catalogue, FieldSpec, Scope
catalogue = Catalogue(
1,
("workload",),
{
"net.topology": FieldSpec("net.topology", binds=True),
"net.region": FieldSpec("net.region", binds=True),
"notes": FieldSpec("notes"),
},
)
scope = Scope(
catalogue,
"workload",
values={"net.topology": "hub", "net.region": "x", "notes": "n"},
sources={"net.region": "assumed"},
)
assert scope.for_context()["binding"] == ["net.topology"]

View File

@@ -0,0 +1,813 @@
"""The Orchestrator as a router (ADR-0007): readings, a disposition, then a playbook.
No language model chooses an action here. Specialists are scripted; the Orchestrator
is the Router, with a scripted decider standing in for Jev, or none at all.
"""
import json
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime, FakeDecider
from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.router import (
REPLY_WRITER,
Router,
RuntimeReplyWriter,
Thresholds,
)
from execution_engine.recorder import InMemoryRecorder
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
SAMPLE_DATA,
WORKFLOW,
DeliverableKind,
sample_submission,
)
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
from tests.scoped import scoped_ledger
K = DeliverableKind
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused by the router.", budget=40),
REPLY_WRITER: RoleSpec(REPLY_WRITER, "Write the reply.", budget=3),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def specialist(kind: DeliverableKind) -> str:
return DEPENDENCY_GRAPH[kind].specialist
def once() -> dict:
return {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
def validation(**changes) -> Call:
"""A validation submission whose verdicts differ from the all-pass sample."""
call = sample_submission(K.VALIDATION_RESULTS)
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
verdicts.update(changes)
return Call(
call.tool,
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
)
def failing(check: str = "K-1") -> Call:
checks = json.loads(SAMPLE_DATA["verdicts.json"])["checks"]
for item in checks:
if item["check"] == check:
item.update(verdict="fail", evidence="min instances is 1")
return validation(checks=checks)
def addition(reason: str) -> Call:
return validation(
proposed_additions=[
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": reason}
]
)
def engine_for(tmp_path, turns: dict, decider=None, writer: bool = False, **options):
runtime = FakeAgentRuntime(turns)
router = Router(
decider or NoDecider(),
RuntimeReplyWriter(runtime, ROLES[REPLY_WRITER]) if writer else None,
Thresholds(),
)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=router,
**options,
)
return engine, runtime
def actions(result) -> list[str]:
return [decision["action"] for decision in result.state["decisions"]]
def reopened(result) -> list[tuple[str, str]]:
return [
(event["subject"], event["detail"])
for event in result.state["ledger"]["events"]
if event["type"] == "reopened"
]
def approve(phase: str) -> dict:
return {"approval": {"phase": phase, "decision": "approve"}}
# --- the playbook on rules alone -------------------------------------------------------
async def test_an_execution_completes_with_no_model_choosing_an_action(tmp_path):
engine, runtime = engine_for(tmp_path, once())
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert "orchestrator" not in {run.role for run in runtime.runs}
assert actions(result) == [
"delegate", # the scope
"request_approval",
"delegate", # requirements
"delegate", # technical decomposition
"request_approval",
"delegate", # product selection
"delegate", # design recommendations
"delegate", # Target architecture
"request_approval",
"delegate", # description and deployment guidance, together
"request_approval",
"delegate", # validation
"proceed",
"request_approval",
"delegate", # solution guide
"request_approval",
"finish",
]
groups = {decision["group"] for decision in result.state["decisions"]}
assert groups == {
"work is ready",
"a Phase is complete",
"validation decided",
"packaging is approved",
}
async def test_every_decision_is_recorded_with_its_turn_group_and_action(tmp_path):
engine, _ = engine_for(tmp_path, once())
result = await engine.start("Ingest webhooks", pre_approved=True)
first = result.state["decisions"][0]
assert first == {
"turn": 1,
"group": "work is ready",
"action": "delegate",
"judged": {},
}
assert [d["turn"] for d in result.state["decisions"]] == sorted(
d["turn"] for d in result.state["decisions"]
)
async def test_when_no_group_applies_a_pre_approved_execution_ends_with_warnings(
tmp_path,
):
turns = {specialist(K.REQUIREMENTS): [[]]} # stops without submitting
engine, _ = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
# The Specialist's question is relayed, answered automatically, and it stops
# again; the router never invents work.
assert result.status == "completed_with_warnings"
# --- Approvals: the summary is written from data, and only data approves ----------------
async def test_the_approval_summary_is_assembled_from_the_data(tmp_path):
engine, _ = engine_for(tmp_path, once())
first = await engine.start("Ingest webhooks")
design = await engine.resume(first.execution_id, approve("requirements_discovery"))
assert first.prompt["approval"]["summary"] == (
"The requirements_discovery Phase is ready: requirements, "
"technical_decomposition submitted. 2 requirements."
)
summary = design.prompt["approval"]["summary"]
assert "Target architecture: 2 Elements, 1 Relationships, 2 Checks." in summary
assert design.prompt["approval"]["target_architecture"]["checks"] == 2
async def test_a_model_writes_the_summary_but_decides_nothing(tmp_path):
turns = {
**once(),
REPLY_WRITER: [
[Call("write_reply", {"text": "The scope, ready."})],
[Call("write_reply", {"text": "Two requirements, ready."})],
],
}
engine, runtime = engine_for(tmp_path, turns, writer=True)
first = await engine.start("Ingest webhooks")
assert first.prompt["approval"]["summary"] == "Two requirements, ready."
# The writer ran once for the scope and once for the requirements.
writer_run = [run for run in runtime.runs if run.role == REPLY_WRITER][-1]
assert writer_run.tools == ("write_reply",)
assert writer_run.task.context["facts"]["deliverables"] == [
"requirements",
"technical_decomposition",
]
async def test_a_writer_that_fails_does_not_stop_the_approval(tmp_path):
engine, _ = engine_for(tmp_path, once(), writer=True) # no writer turn scripted
first = await engine.start("Ingest webhooks")
assert first.prompt["approval"]["summary"].startswith(
"The requirements_discovery Phase is ready"
)
async def test_a_text_reply_never_approves_and_the_approval_is_asked_again(tmp_path):
decider = FakeDecider(
{
"asks_change": Answer(0.03, 0.94),
"change_target": Answer("requirements", 0.5),
"change_clarity": Answer(0.0, 0.9),
}
)
engine, _ = engine_for(tmp_path, once(), decider)
first = await engine.start("Ingest webhooks")
again = await engine.resume(first.execution_id, {"reply": "Looks good to me."})
assert again.prompt["approval"]["phase"] == "requirements_discovery"
assert again.state["ledger"]["phases"]["requirements_discovery"] != "approved"
# --- validation: the code, the graph, or proceed ----------------------------------------
async def test_a_failed_check_sends_the_code_back_and_leaves_the_design_approved(
tmp_path,
):
turns = once()
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [
[failing("K-1")],
[sample_submission(K.VALIDATION_RESULTS)],
]
engine, runtime = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert reopened(result) == [
("deployment_guidance", "K-1 failed: min instances is 1")
]
assert "revise_code" in actions(result)
architect_runs = [
run for run in runtime.runs if run.role == specialist(K.TARGET_ARCHITECTURE)
]
assert len(architect_runs) == 1
async def test_a_check_the_code_cannot_meet_goes_back_to_the_graph_unvalidated(
tmp_path,
):
"""The code's author found the Check names a setting the resource lacks. No
validation is spent on it: the Target architecture is revised first."""
cannot = sample_submission(K.DEPLOYMENT_GUIDANCE)
implementation = json.loads(SAMPLE_DATA["implementation.json"])
implementation[1] = {"check": "K-2", "unimplementable": "no such argument"}
turns = once()
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [
[
Call(
cannot.tool,
{
**cannot.args,
"attachments": {
**cannot.args["attachments"],
"implementation.json": json.dumps(implementation),
},
},
)
],
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
]
engine, runtime = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert reopened(result) == [
(
"target_architecture",
(
"Check K-2 cannot be implemented as written (no such argument). "
"Rewrite it so it can be met, or remove it with a reason. The same "
"fault may be in other Checks: read every Check against each reason "
"given here and correct every one it applies to."
),
)
]
assert "the code cannot meet a Check" in {
decision["group"] for decision in result.state["decisions"]
}
validations = [
run for run in runtime.runs if run.role == specialist(K.VALIDATION_RESULTS)
]
assert len(validations) == 1
async def test_a_check_that_fails_twice_running_is_sent_back_to_the_graph(tmp_path):
"""The code was revised and the same Check failed again: the Check is suspect."""
turns = once()
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 3
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [
[failing("K-1")],
[failing("K-1")],
[sample_submission(K.VALIDATION_RESULTS)],
]
engine, _ = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
[first, second] = reopened(result)
assert first[0] == "deployment_guidance"
assert second[0] == "target_architecture"
assert "K-1 failed in two validations in a row" in second[1]
assert actions(result).count("revise_code") == 1
async def test_without_jev_a_gap_naming_a_real_requirement_revises_the_graph(tmp_path):
turns = once()
for kind in (
K.TARGET_ARCHITECTURE,
K.ARCHITECTURE_DESCRIPTION,
K.DEPLOYMENT_GUIDANCE,
):
turns[specialist(kind)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [
[addition("NFR-1 has no Check on timeouts")],
[sample_submission(K.VALIDATION_RESULTS)],
]
engine, _ = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
[(kind, reason)] = reopened(result)
assert kind == "target_architecture"
assert reason == (
"Add a Check on E-1: A timeout is set. (serves NFR-1 has no Check on timeouts)."
)
async def test_without_jev_a_gap_that_names_nothing_real_is_left(tmp_path):
turns = once()
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("it would be nice")]]
engine, _ = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert reopened(result) == []
async def test_jev_judges_whether_a_gap_is_worth_a_revision(tmp_path):
turns = once()
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("NFR-1")]]
immaterial = FakeDecider({"gap_material": Answer(0.04, 0.92)})
engine, _ = engine_for(tmp_path, turns, immaterial)
result = await engine.start("Ingest webhooks", pre_approved=True)
# The rule alone would have revised the graph; the reading says it is not needed.
assert reopened(result) == []
judged = next(d for d in result.state["decisions"] if d["action"] == "proceed")
assert judged["judged"] == {"gap_material": {"value": 0.04, "confidence": 0.92}}
[(state, questions, scope)] = immaterial.asked
assert "A timeout is set." in state and scope == result.execution_id
assert questions["gap_material"].kind == "noul"
async def test_an_uncertain_gap_is_put_to_the_user_who_decides(tmp_path):
turns = once()
for kind in (
K.TARGET_ARCHITECTURE,
K.ARCHITECTURE_DESCRIPTION,
K.DEPLOYMENT_GUIDANCE,
):
turns[specialist(kind)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [
[addition("NFR-1")],
[sample_submission(K.VALIDATION_RESULTS)],
]
decider = FakeDecider(
{
"gap_material": Answer(0.5, 0.0), # the grey zone
"wants_revision": Answer(0.96, 0.92),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
result = await engine.start("Ingest webhooks")
for phase in (
"requirements_discovery",
"solution_architecture",
"solution_implementation",
):
result = await engine.resume(result.execution_id, approve(phase))
[question] = result.prompt["questions"]
assert (
"found gaps in the Target architecture: A timeout is set."
in (question["question"])
)
result = await engine.resume(result.execution_id, {"reply": "Yes, revise it."})
assert reopened(result)[0][0] == "target_architecture"
# Reopening the graph withdrew the design Approval, so it is asked for again.
assert result.prompt["approval"]["phase"] == "solution_architecture"
# --- the user's message: act, queue or close --------------------------------------------
def to_design_approval(engine):
async def run():
first = await engine.start("Ingest webhooks")
return await engine.resume(
first.execution_id, approve("requirements_discovery")
)
return run()
def changes(phase: str, feedback: str) -> dict:
return {"approval": {"phase": phase, "decision": "changes", "feedback": feedback}}
async def test_a_requested_change_is_routed_to_the_deliverable_it_affects(tmp_path):
turns = once()
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
decider = FakeDecider(
{
"change_target": Answer("design_recommendations", 0.88),
"change_clarity": Answer(1.8, 0.8),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
design = await to_design_approval(engine)
result = await engine.resume(
design.execution_id,
changes("solution_architecture", "Use customer-managed keys on the topic."),
)
assert reopened(result) == [
("design_recommendations", "Use customer-managed keys on the topic.")
]
# Product selection was not touched; the design is asked for again once redone.
assert result.state["ledger"]["deliverables"]["product_selection"]["revisions"] == 0
assert result.prompt["approval"]["phase"] == "solution_architecture"
[(state, questions, _)] = decider.asked
assert "The user's message: Use customer-managed keys on the topic." in state
assert questions["change_target"].options == (
"scope",
"requirements",
"technical_decomposition",
"product_selection",
"design_recommendations",
"target_architecture",
)
assert "asks_change" not in questions # a structured request is known to be one
async def test_an_unclear_change_is_confirmed_with_the_user_before_anything_moves(
tmp_path,
):
turns = once()
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
decider = FakeDecider(
{
"change_target": Answer("product_selection", 0.31), # not relied on
"change_clarity": Answer(1.5, 0.7),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
design = await to_design_approval(engine)
asked = await engine.resume(
design.execution_id, changes("solution_architecture", "Tighten it up.")
)
assert reopened(asked) == []
[question] = asked.prompt["questions"]
assert question["question"].startswith("Do you want product_selection changed?")
result = await engine.resume(
asked.execution_id,
{"reply": "No, change the target architecture: add a Check on timeouts."},
)
[(kind, reason)] = reopened(result)
assert kind == "target_architecture"
assert "Tighten it up." in reason and "add a Check on timeouts" in reason
async def test_a_free_text_reply_that_asks_for_a_change_is_acted_on(tmp_path):
turns = once()
turns[specialist(K.REQUIREMENTS)] = [
[sample_submission(K.REQUIREMENTS)],
[sample_submission(K.REQUIREMENTS)],
]
turns[specialist(K.TECHNICAL_DECOMPOSITION)] *= 2
decider = FakeDecider(
{
"asks_change": Answer(0.93, 0.86),
"change_target": Answer("requirements", 0.9),
"change_clarity": Answer(1.9, 0.8),
"already_agreed": Answer(0.03, 0.94),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
first = await engine.start("Ingest webhooks")
result = await engine.resume(
first.execution_id, {"reply": "Retention must be 7 years, not 30 days."}
)
assert reopened(result) == [
("requirements", "Retention must be 7 years, not 30 days.")
]
async def test_without_jev_a_change_is_acted_on_only_when_the_deliverable_is_named(
tmp_path,
):
turns = once()
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
engine, _ = engine_for(tmp_path, turns)
design = await to_design_approval(engine)
unnamed = await engine.resume(
design.execution_id, changes("solution_architecture", "Make it cheaper.")
)
assert reopened(unnamed) == []
assert unnamed.prompt["questions"][0]["question"].startswith(
"Which Deliverable should change"
)
named = await engine.resume(
unnamed.execution_id, {"reply": "The design recommendations."}
)
[(kind, reason)] = reopened(named)
assert kind == "design_recommendations"
assert reason == (
"Make it cheaper.\nThe user then said: The design recommendations."
)
# --- permission is the Ledger's, whatever was decided ------------------------------------
async def test_past_the_revision_limit_the_user_decides(tmp_path):
turns = once()
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]]
engine, _ = engine_for(tmp_path, turns, max_revisions=1)
result = await engine.start("Ingest webhooks")
for phase in (
"requirements_discovery",
"solution_architecture",
"solution_implementation",
"solution_implementation",
):
result = await engine.resume(result.execution_id, approve(phase))
assert len(reopened(result)) == 1
[question] = result.prompt["questions"]
assert (
"deployment_guidance has reached its revision limit (1)"
in (question["question"])
)
# --- the two faults of evaluation case 04 (2026-10-08) -----------------------------------
async def test_a_reply_at_the_revision_limit_is_the_answer_and_the_run_carries_on(
tmp_path,
):
"""The user is told once. Their reply is not read as a new request for changes,
the Deliverable is not reopened again, and nothing is asked a second time."""
turns = once()
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]]
# A decider that would call any reply a clear request to change the code.
decider = FakeDecider(
{
"asks_change": Answer(0.95, 0.9),
"change_target": Answer("deployment_guidance", 0.95),
"change_clarity": Answer(1.9, 0.8),
}
)
engine, _ = engine_for(tmp_path, turns, decider, max_revisions=1)
result = await engine.start("Ingest webhooks")
for phase in (
"requirements_discovery",
"solution_architecture",
"solution_implementation",
"solution_implementation",
):
result = await engine.resume(result.execution_id, approve(phase))
[question] = result.prompt["questions"]
assert "I will carry on with it as it stands" in question["question"]
result = await engine.resume(
result.execution_id, {"reply": "Please change the code again."}
)
# It moves on to the Approval of validation; the code was reopened once only.
assert result.prompt["approval"]["phase"] == "solution_validation"
assert len(reopened(result)) == 1
assert result.state["revision_limit_asked"] == ["deployment_guidance"]
assert decider.asked == [] # the reply was never put to the decider
async def test_only_the_users_own_words_are_read_as_their_message(tmp_path):
"""A reply that quotes our question back must not be judged on our words."""
question = "Please confirm the requirements draft. Shall I change anything?"
turns = once()
turns[specialist(K.REQUIREMENTS)] = [
[Call("needs_input", {"questions": [question]})],
[sample_submission(K.REQUIREMENTS)],
]
decider = FakeDecider(
{
"asks_change": Answer(0.05, 0.9),
"change_target": Answer("requirements", 0.5),
"change_clarity": Answer(0.2, 0.5),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
first = await engine.start("Ingest webhooks")
result = await engine.resume(
first.execution_id, {"reply": f"Q: {question}\nA: Yes, approved."}
)
# Nothing has been submitted yet, so nothing is judged on that turn. Let the
# next reply, to the decomposer's approval prompt, be a text one.
result = await engine.resume(result.execution_id, {"reply": "Looks fine."})
assert result.state["last_reply"] is None
state, _, _ = decider.asked[-1]
assert state.endswith("The user's message: Looks fine.")
def test_a_quoted_question_is_removed_from_what_the_user_said():
from execution_engine.nodes.execution_nodes import _own_words
asked = ["Please confirm the draft. Shall I change the design?", "Which region?"]
reply = (
"Q: Please confirm the draft. Shall I change the design?\nA: Yes, approved.\n\n"
"Q: Which region?\nA: europe-west2"
)
assert _own_words(reply, asked) == "Yes, approved.\neurope-west2"
assert _own_words("Make retention 7 years.", asked) == "Make retention 7 years."
async def test_the_questions_asked_are_given_to_the_decider_as_background(tmp_path):
from execution_engine.nodes.router import TurnView
ledger = scoped_ledger()
ledger.start([K.REQUIREMENTS], actor="orchestrator")
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
decider = FakeDecider(
{
"asks_change": Answer(0.05, 0.9),
"change_target": Answer("requirements", 0.5),
"change_clarity": Answer(0.2, 0.5),
}
)
router = Router(decider, None, Thresholds())
view = TurnView(
execution_id="e1",
pre_approved=False,
ledger=ledger,
deliverables={},
pending_questions=[],
outcomes=[],
last_reply={
"kind": "reply",
"text": "Yes, approved.",
"asked": ["Shall I change the design?"],
},
)
from execution_engine.nodes.router import Readings
await router._read_message(view, Readings())
state, _, _ = decider.asked[0]
assert "replying to these questions from the agents:\n- Shall I change" in state
assert state.endswith("The user's message: Yes, approved.")
async def test_asking_the_user_again_and_again_with_nothing_done_stops_the_run(
tmp_path,
):
"""Whatever causes it, a run that only asks is stopped, not left to the turn limit."""
from execution_engine.nodes.execution_nodes import MAX_IDLE_ASKS
from execution_engine.nodes.router import TurnResult
class OnlyAsks:
"""An Orchestrator that asks the user something on every turn."""
async def turn(self, view, tools):
ask = next(tool for tool in tools if tool.name == "ask_user")
reply = await ask.handler(questions=["What now?"])
return TurnResult(reply.outcome)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=FakeAgentRuntime({}),
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=OnlyAsks(),
)
result = await engine.start("Ingest webhooks")
asked = 1
while result.status == "awaiting_user" and asked < 20:
result = await engine.resume(result.execution_id, {"reply": "Carry on."})
asked += 1
assert result.status == "completed_with_warnings"
assert asked == MAX_IDLE_ASKS + 1
assert "asked" in result.state["warnings"][-1]
assert "no work done in between" in result.state["warnings"][-1]
async def test_a_reply_that_restates_what_is_agreed_is_an_answer_not_a_change(
tmp_path,
):
"""Evaluation case 04: a requirement said again in reply to a later question."""
from execution_engine.nodes.router import Readings, TurnView
ledger = scoped_ledger()
ledger.start([K.REQUIREMENTS], actor="orchestrator")
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
view = TurnView(
execution_id="e1",
pre_approved=False,
ledger=ledger,
deliverables={
"requirements": {
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
}
},
pending_questions=[],
outcomes=[],
last_reply={"kind": "reply", "text": "99.9% availability.", "asked": []},
)
asks = {
"asks_change": Answer(0.9, 0.8),
"change_target": Answer("requirements", 0.95),
"change_clarity": Answer(1.9, 0.8),
}
restated, changed = Readings(), Readings()
decider = FakeDecider({**asks, "already_agreed": Answer(0.85, 0.7)})
await Router(decider, None, Thresholds())._read_message(view, restated)
await Router(
FakeDecider({**asks, "already_agreed": Answer(0.02, 0.96)}), None, Thresholds()
)._read_message(view, changed)
assert restated.change == "close"
assert changed.change == "act"
state, questions, _ = decider.asked[0]
assert "The agreed requirements:" in state and "NFR-1" in state
assert "already_agreed" in questions

View File

@@ -0,0 +1,126 @@
"""Which references a Specialist is told to read first (iteration 5 step 8)."""
from pathlib import Path
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import FakeAgentRuntime, FakeDecider
from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from execution_engine.tools.skill_routing import read_first, references_of, routable
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
WORKFLOW,
sample_submission,
)
from tests.reference_workflow.workflow import DeliverableKind as K
PATTERNS = {
"references/cloud-sql.md": "Cloud SQL. A private PostgreSQL instance.",
"references/pubsub.md": "Pub/Sub. A topic and a subscription.",
}
def skill(root: Path, name: str, references: dict[str, str]) -> Path:
folder = root / name
(folder / "references").mkdir(parents=True)
(folder / "SKILL.md").write_text(f"---\nname: {name}\ndescription: d\n---\n")
for file, text in references.items():
(folder / "references" / file).write_text(text)
return folder
def test_a_reference_is_described_by_its_title_and_first_paragraph(tmp_path):
folder = skill(
tmp_path,
"patterns",
{"kms.md": "# Cloud KMS\n\nA key ring and a key.\n\n- A rule.\n"},
)
assert references_of(folder) == {
"references/kms.md": "Cloud KMS. A key ring and a key"
}
def test_only_a_skill_with_enough_references_is_routed_within(tmp_path):
many = skill(tmp_path, "many", {f"r{n}.md": f"# R{n}\n\nText.\n" for n in range(4)})
few = skill(tmp_path, "few", {"one.md": "# One\n\nText.\n"})
none = tmp_path / "none"
none.mkdir()
assert list(routable({"many": many, "few": few, "none": none})) == ["many"]
async def test_the_references_a_task_needs_are_named():
decider = FakeDecider(
{
"patterns/references/cloud-sql.md": Answer(1.8, 0.9),
"patterns/references/pubsub.md": Answer(0.1, 0.9),
}
)
chosen = await read_first(decider, "Write the code.", {"patterns": PATTERNS})
assert chosen == [{"skill": "patterns", "resource": "references/cloud-sql.md"}]
_, questions, _ = decider.asked[0]
assert "Cloud SQL. A private PostgreSQL instance." in (
questions["patterns/references/cloud-sql.md"].instructions
)
async def test_without_a_decision_nothing_is_named():
assert await read_first(NoDecider(), "Write.", {"patterns": PATTERNS}) is None
async def test_a_specialist_with_routable_skills_is_told_where_to_start(tmp_path):
roles = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
seen: dict[str, object] = {}
def recording(kind):
def turn(task):
seen[kind.value] = task.context["read_first"]
return [sample_submission(kind)]
return turn
decider = FakeDecider(
{
"patterns/references/cloud-sql.md": Answer(0.2, 0.9),
"patterns/references/pubsub.md": Answer(1.9, 0.9),
}
)
author = DEPENDENCY_GRAPH[K.DEPLOYMENT_GUIDANCE].specialist
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=FakeAgentRuntime(
{
spec.specialist: [recording(kind)]
for kind, spec in DEPENDENCY_GRAPH.items()
}
),
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
decider=decider,
skill_references={author: {"patterns": PATTERNS}},
)
await engine.start("Ingest webhooks", pre_approved=True)
assert seen[K.DEPLOYMENT_GUIDANCE.value] == [
{"skill": "patterns", "resource": "references/pubsub.md"}
]
assert seen[K.REQUIREMENTS.value] is None
state, _, _ = decider.asked[0]
assert state.startswith("Write the deployment guidance Deliverable")
assert "Elements of the Target architecture:" in state

View File

@@ -0,0 +1,306 @@
"""Two fixes from the run cancelled on 2026-10-07.
The validator wrote every verdict in one reply and timed out five times; it now
records them a few at a time. And two Checks named settings their resource types do
not have, each costing a Revision; the design is now checked against the provider's
schema, in code, before it is submitted.
"""
import json
from execution_engine.adk.runtime import Submitted, ToolResult
from execution_engine.tools.execution import specialist_tools
from tests.reference_workflow.local import TerraformRunner
from tests.reference_workflow.schema_review import Reviews, SchemaReview, named_settings
from tests.reference_workflow.workflow import (
SAMPLE_DATA,
WORKFLOW,
DeliverableKind,
sample_submission,
)
K = DeliverableKind
CODE = {"terraform/main.tf": "# v1\n", "implementation.json": "[]"}
VERDICTS = json.loads(SAMPLE_DATA["verdicts.json"])
CONTENT = sample_submission(K.VALIDATION_RESULTS).args["content"]
def upstream(code=CODE) -> dict:
return {
"target_architecture": {
"attachments": {
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
}
},
"deployment_guidance": {"attachments": dict(code)},
}
def validator_tools(working: dict, code=CODE) -> dict:
return {
tool.name: tool
for tool in specialist_tools(
WORKFLOW,
K.VALIDATION_RESULTS,
(),
allow_cloud_read=False,
upstream=upstream(code),
working=working,
)
}
# --- verdicts a few at a time ------------------------------------------------------------
async def test_verdicts_are_recorded_a_few_at_a_time_and_attached_on_submission():
working: dict = {}
tools = validator_tools(working)
first = await tools["verdict_put"].handler(
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
)
second = await tools["verdict_put"].handler(checks=VERDICTS["checks"][1:])
result = await tools["submit_deliverable"].handler(content=CONTENT)
assert first.message == "Recorded. 1 of 2 Checks have a verdict. Left: K-2."
assert second.message == "Recorded. All 2 Checks have a verdict."
assert isinstance(result.outcome, Submitted)
attached = json.loads(result.outcome.attachments["verdicts.json"])
assert [item["check"] for item in attached["checks"]] == ["K-1", "K-2"]
assert attached["dry_run"]["verdict"] == "pass"
async def test_a_submission_with_a_check_left_out_names_it():
tools = validator_tools({})
await tools["verdict_put"].handler(
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
)
result = await tools["submit_deliverable"].handler(content=CONTENT)
assert result.outcome is None
assert "Check K-2 is not accounted for" in result.message
async def test_a_verdict_given_again_replaces_the_first_and_additions_are_kept():
working: dict = {}
tools = validator_tools(working)
addition = {
"statement": "A timeout is set.",
"applies_to": "E-1",
"reason": "NFR-1",
}
await tools["verdict_put"].handler(checks=VERDICTS["checks"])
await tools["verdict_put"].handler(
checks=[{"check": "K-1", "verdict": "fail", "evidence": "min is 1"}],
proposed_additions=[addition, addition],
)
read = await tools["verdict_read"].handler()
recorded = json.loads(read.message.rsplit("\n", 1)[0])
assert [(c["check"], c["verdict"]) for c in recorded["checks"]] == [
("K-1", "fail"),
("K-2", "pass"),
]
assert recorded["proposed_additions"] == [addition]
assert read.message.endswith("All 2 Checks have a verdict.")
async def test_a_run_cut_short_carries_on_from_what_it_recorded():
working: dict = {}
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"][:1])
# The same code, a later attempt: the working memory came back with the Execution.
read = await validator_tools(working)["verdict_read"].handler()
assert "1 of 2 Checks have a verdict. Left: K-2." in read.message
async def test_verdicts_on_code_that_has_changed_are_dropped():
working: dict = {}
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"])
revised = {**CODE, "terraform/main.tf": "# v2\n"}
read = await validator_tools(working, revised)["verdict_read"].handler()
assert read.message.startswith("No verdicts recorded yet.")
async def test_verdicts_passed_as_an_attachment_still_work():
call = sample_submission(K.VALIDATION_RESULTS)
result = await validator_tools({})["submit_deliverable"].handler(**call.args)
assert isinstance(result.outcome, Submitted)
def test_only_the_validator_records_verdicts():
assert "verdict_put" in validator_tools({})
assert "verdict_put" not in {
tool.name
for tool in specialist_tools(
WORKFLOW, K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False
)
}
# --- the design against the provider's schema --------------------------------------------
def test_the_settings_a_check_names_are_read_from_its_statement():
assert named_settings("`kms_key_name` is set on the topic") == ("kms_key_name",)
assert named_settings("template.scaling.min_instance_count is at least 2") == (
"template.scaling.min_instance_count",
)
assert named_settings("startup_cpu_boost is true") == ("startup_cpu_boost",)
# A value in backticks is not a setting.
assert named_settings("`enable_cdn` is `false`") == ("enable_cdn",)
assert named_settings("`deletion_protection` is `true`") == ("deletion_protection",)
# Values, roles and plain prose name no setting.
assert named_settings("ingress is INGRESS_TRAFFIC_ALL") == ()
assert named_settings("roles/run.invoker is granted to allUsers") == ()
assert named_settings('the filter contains "oldest_unacked_message_age"') == ()
assert named_settings("The service is private.") == ()
SCHEMA = {
"cloud_service": (
"example/cloud 1.2.3",
{
"attributes": {"name": {}, "ingress": {}},
"block_types": {
"template": {
"block": {
"attributes": {"service_account": {}},
"block_types": {
"scaling": {
"block": {"attributes": {"min_instance_count": {}}}
}
},
}
}
},
},
),
"cloud_job": ("example/cloud 1.2.3", {"attributes": {"name": {}}}),
"cloud_service_iam_member": ("example/cloud 1.2.3", {"attributes": {"role": {}}}),
}
class Runner(TerraformRunner):
def __init__(self, schemas=SCHEMA) -> None:
super().__init__()
self._given = schemas
async def _load_schemas(self):
return self._given
def graph(checks, resource_types=("cloud_service", "cloud_job")) -> dict[str, str]:
elements = [
{
"id": f"E-{n}",
"name": f"Thing {n}",
"product": "Example",
"resource_type": kind,
}
for n, kind in enumerate(resource_types, start=1)
]
data = {
"elements": elements,
"relationships": [{"id": "R-1", "from": "E-1", "to": "E-2", "kind": "calls"}],
"checks": [
{"id": f"K-{n}", "applies_to": on, "statement": text, "source": "NFR-1"}
for n, (on, text) in enumerate(checks, start=1)
],
}
return {"target-architecture.json": json.dumps(data)}
async def test_a_check_naming_a_setting_its_resource_type_lacks_is_raised():
design = graph(
[
("E-1", "template.scaling.min_instance_count is at least 2"),
("E-2", "`min_instance_count` is at least 2"),
("E-2", "the name is set"),
]
)
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
assert len(raised) == 1
assert raised[0].startswith(
"Check K-2: cloud_job has no setting named min_instance_count"
)
async def test_a_setting_found_anywhere_in_the_resource_type_is_accepted():
design = graph(
[
("E-1", "service_account is the dedicated identity"),
("E-1", "ingress is INTERNAL"),
("E-1", "a cloud_service_iam_member grants the invoker role"),
("R-1", "any_made_up_setting is checked on a Relationship"),
("E-2", "the name is set"),
]
)
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) == ()
async def test_a_value_written_with_dots_is_not_read_as_a_setting():
"""A database flag or a service name is a value. A path is a setting only when
it starts at a name the resource type has."""
design = graph(
[
("E-1", "the flag cloudsql.iam_authentication is on"),
("E-1", "audit logs are on for run.googleapis.com"),
("E-1", "template.scaling.made_up_count is at least 2"),
("E-2", "the name is set"),
]
)
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
assert len(raised) == 1 and "template.scaling.made_up_count" in raised[0]
async def test_a_resource_type_the_provider_lacks_is_raised_with_close_matches():
design = graph(
[("E-1", "the name is set"), ("E-2", "the name is set")],
resource_types=("cloud_servce", "cloud_job"),
)
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
assert len(raised) == 1
assert raised[0].startswith(
"Element E-1: cloud_servce is not a resource type of the provider."
)
assert "Close matches: cloud_service" in raised[0]
async def test_no_schema_and_other_deliverables_raise_nothing():
design = graph([("E-1", "made_up_thing is set"), ("E-2", "the name is set")])
unavailable = Runner(ToolResult("Schema lookup is unavailable."))
assert await SchemaReview(unavailable)(K.TARGET_ARCHITECTURE, design, None) == ()
assert await SchemaReview(Runner())(K.DEPLOYMENT_GUIDANCE, design, None) == ()
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, {}, None) == ()
async def test_several_reviews_raise_together_and_one_failing_loses_nothing():
async def one(kind, attachments, upstream, scope=""):
return ["first"]
async def broken(kind, attachments, upstream, scope=""):
raise RuntimeError("down")
async def two(kind, attachments, upstream, scope=""):
return ("second",)
raised = await Reviews([one, broken, two])(K.TARGET_ARCHITECTURE, {}, None)
assert raised == ("first", "second")

View File

@@ -0,0 +1,314 @@
"""The work breakdown structure: a plan checked, scheduled and drawn by code."""
import json
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.runtime import NoDecider, RoleSpec
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from tests.reference_workflow.work_breakdown import (
check_work_breakdown,
mermaid,
render,
schedule,
summary,
)
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
SAMPLE_DATA,
WORKFLOW,
sample_submission,
)
from tests.reference_workflow.workflow import DeliverableKind as K
GRAPH = SAMPLE_DATA["target-architecture.json"] # Elements E-1 and E-2
PLANNER = DEPENDENCY_GRAPH[K.WORK_BREAKDOWN].specialist
def upstream(**delivery) -> dict:
scope = {"stated": {"delivery": delivery}, "assumed": []}
return {
"target_architecture": {"attachments": {"target-architecture.json": GRAPH}},
"scope": {"attachments": {"scope.json": json.dumps(scope)}},
}
def plan(packages=None, activities=None) -> dict[str, str]:
data = json.loads(SAMPLE_DATA["work-breakdown.json"])
if packages is not None:
data["work_packages"] = packages
if activities is not None:
data["activities"] = activities
return {"work-breakdown.json": json.dumps(data)}
def activity(ident: str, days: float, *after: str, **more) -> dict:
return {
"id": ident,
"work_package": "WP-1",
"name": f"Do {ident}",
"days": days,
"after": list(after),
**more,
}
PACKAGE = json.loads(SAMPLE_DATA["work-breakdown.json"])["work_packages"][0]
# --- the plan is checked against the design ---------------------------------------------
def test_a_plan_that_covers_the_design_is_accepted():
assert check_work_breakdown(plan(), upstream()) == []
def test_every_element_of_the_design_belongs_to_a_work_package():
only_one = [{**PACKAGE, "elements": ["E-1"]}]
invented = [{**PACKAGE, "elements": ["E-1", "E-2", "E-9"]}]
(left_out,) = check_work_breakdown(plan(only_one), upstream())
(unknown,) = check_work_breakdown(plan(invented), upstream())
assert "no work package delivers E-2" in left_out
assert "WP-1 delivers E-9, which is not an Element" in unknown
def test_a_plan_that_cannot_be_scheduled_is_refused_with_the_reason():
def problems(*activities):
return check_work_breakdown(plan(activities=list(activities)), upstream())
assert any(
"depend on each other in a circle" in p
for p in problems(activity("A-1", 1, "A-2"), activity("A-2", 1, "A-1"))
)
assert any(
"comes after A-9, which is not an activity" in p
for p in problems(activity("A-1", 1, "A-9"))
)
assert any(
"needs `days`, a number above 0" in p for p in problems(activity("A-1", 0))
)
assert any("split anything over" in p for p in problems(activity("A-1", 400)))
assert any(
"owner must be one of" in p
for p in problems(activity("A-1", 1, owner="vendor"))
)
assert any(
"is in WP-7, not a work package" in p
for p in problems({**activity("A-1", 1), "work_package": "WP-7"})
)
def test_a_work_package_holds_work_or_other_work_packages():
parent = {**PACKAGE, "id": "WP-0", "elements": []}
child = {**PACKAGE, "parent": "WP-0"}
idle = {**PACKAGE, "id": "WP-2", "elements": []}
assert check_work_breakdown(plan([parent, child]), upstream()) == []
assert any(
"WP-2 has no activity and no work package beneath it" in p
for p in check_work_breakdown(
plan([child | {"parent": None}, idle]), upstream()
)
)
assert any(
"has parent WP-9, which does not exist" in p
for p in check_work_breakdown(plan([{**PACKAGE, "parent": "WP-9"}]), upstream())
)
# --- the schedule is computed -----------------------------------------------------------
def test_an_activity_starts_when_what_it_follows_has_finished():
placed = schedule(
[activity("A-1", 2), activity("A-2", 3, "A-1"), activity("A-3", 1, "A-2")]
)
assert [(p.start, p.finish) for p in placed.values()] == [(0, 2), (2, 5), (5, 6)]
assert all(p.critical for p in placed.values())
def test_independent_work_runs_at_once_up_to_the_size_of_the_team():
work = [activity("A-1", 4), activity("A-2", 4), activity("A-3", 4)]
two = schedule(work, team_size=2)
three = schedule(work, team_size=3)
assert max(p.finish for p in two.values()) == 8
assert sorted(p.start for p in two.values()) == [0, 0, 4]
assert max(p.finish for p in three.values()) == 4
def test_what_the_customer_owns_takes_nobody_from_the_team():
work = [
activity("A-1", 10, owner="customer"),
activity("A-2", 3),
activity("A-3", 2, "A-1", "A-2"),
]
placed = schedule(work, team_size=1)
# The lead time and the team's work run together; the end waits on the longer.
assert placed["A-2"].start == 0 and placed["A-3"].start == 10
assert placed["A-1"].critical and placed["A-3"].critical
assert not placed["A-2"].critical
def test_when_people_are_short_the_longest_chain_goes_first():
work = [activity("A-1", 1), activity("A-2", 1), activity("A-3", 5, "A-2")]
placed = schedule(work, team_size=1)
assert placed["A-2"].start == 0 and placed["A-1"].start > 0
# --- the views are written from the plan ------------------------------------------------
def test_the_three_views_are_written_from_the_data():
text = render(plan(), upstream(team_size=2))
assert "## WBS dictionary" in text
# Acceptance is the Checks on the work package's Elements, read from the design.
assert (
"| WP-1 | Ingress | The service and its identity. | a running service | E-1, E-2 | K-1, K-2 |"
in text
)
assert (
"| A-2 | Deploy the service | WP-1 | 2 | A-1 | Day 1 | Day 3 | team | yes |"
in text
)
assert "## Timeline" in text and "Day 0" in text
assert "3 working days, about 0.6 weeks, with a team of 2 and a 5-day week." in text
assert "indicative" in text
def test_a_plan_over_the_delivery_window_says_so_and_names_the_critical_path():
long = plan(activities=[activity("A-1", 20), activity("A-2", 15, "A-1")])
over = render(long, upstream(team_size=2, window_weeks=4))
inside = render(long, upstream(team_size=2, window_weeks=10))
assert "**That is 3.0 weeks over the 4-week delivery window.**" in over
assert "The critical path is A-1 -> A-2" in over
assert "inside the 10-week delivery window, with 3.0 weeks to spare" in inside
assert summary(long, upstream(window_weeks=4))["within_window"] is False
def test_an_assumed_team_size_is_said_to_be_assumed():
assert "(assumed; the scope gives no team size)" in render(plan(), upstream())
def test_a_customers_lead_time_is_shown_as_theirs():
waiting = plan(
activities=[
activity(
"A-1", 30, owner="customer", assumption="The carrier takes six weeks."
),
activity("A-2", 2, "A-1"),
]
)
text = render(waiting, upstream(team_size=2))
assert "The customer owns these, and the plan waits on them:" in text
assert "- A-1 Do A-1: 30 days. The carrier takes six weeks." in text
assert "░" in text and "█" in text
def test_calendar_dates_and_a_chart_appear_only_with_a_start_date():
dated = upstream(team_size=2, target_start_date="2026-11-02") # a Monday
assert mermaid(plan(), upstream(team_size=2)) == ""
assert "Dates" not in render(plan(), upstream(team_size=2))
chart = mermaid(plan(), dated)
assert chart.startswith("gantt\n dateFormat YYYY-MM-DD")
assert "Deploy the service :crit, A-2, 2026-11-03, 2d" in chart
assert "| 2026-11-03 to 2026-11-04 |" in render(plan(), dated)
def test_a_week_of_working_days_skips_the_weekend():
week = plan(activities=[activity("A-1", 5), activity("A-2", 1, "A-1")])
text = render(week, upstream(team_size=1, target_start_date="2026-11-02"))
# Five working days from Monday end on Friday; the next starts on Monday.
assert "| 2026-11-02 to 2026-11-06 |" in text
assert "| 2026-11-09 to 2026-11-09 |" in text
# --- in an Execution --------------------------------------------------------------------
async def test_an_execution_delivers_a_plan_and_adds_its_timeline_to_the_guide(
tmp_path,
):
roles = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
runtime = FakeAgentRuntime(
{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
result = await engine.start("Ingest", customer_case="acme", pre_approved=True)
assert result.status == "completed"
out = next(tmp_path.rglob("execution.json")).parent
breakdown = (out / "deliverables" / "work_breakdown.md").read_text()
assert "## WBS dictionary" in breakdown and "## Activity sequence" in breakdown
assert (out / "work-breakdown.json").is_file()
guide = (out / "solution-architecture-guide.md").read_text()
assert "## Delivery plan" in guide and "A-2 Deploy the service" in guide
manifest = json.loads((out / "execution.json").read_text())
assert manifest["delivery_plan"]["working_days"] == 3
assert manifest["delivery_plan"]["critical_path"] == ["A-1", "A-2"]
# The planner was given the design and the code to plan from, in full.
planner = next(run for run in runtime.runs if run.role == PLANNER)
given = planner.task.context["dependencies"]
assert "content" in given["target_architecture"]
assert "content" in given["deployment_guidance"]
async def test_a_plan_that_leaves_an_element_out_is_returned_to_the_planner(tmp_path):
from execution_engine.tools.execution import specialist_tools
(submit,) = (
tool
for tool in specialist_tools(
WORKFLOW, K.WORK_BREAKDOWN, (), allow_cloud_read=False, upstream=upstream()
)
if tool.name == "submit_deliverable"
)
partial = Call(
"submit_deliverable",
{
"content": "## Approach\nIn order.\n",
"attachments": plan([{**PACKAGE, "elements": ["E-1"]}]),
},
)
refused = await submit.handler(**partial.args)
assert refused.outcome is None
assert "no work package delivers E-2" in refused.message

View File

@@ -0,0 +1,490 @@
"""A Specialist's working copy, and what each Specialist is given to read.
Three things made an Execution slow: a Specialist rewrote a whole Deliverable in one
reply, sent every file again on each tool call, and read every earlier Deliverable in
full. A working copy and trimmed reading remove each of them.
"""
import json
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.runtime import NoDecider, RoleSpec, Submitted, ToolResult
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from execution_engine.tools.execution import specialist_tools
from execution_engine.tools.toolkit import make_tool_factory
from execution_engine.tools.workspace import file_tools, graph_tools
from tests.reference_workflow.hooks import GRAPH_SHAPE
from tests.reference_workflow.local import TerraformRunner, local_tools
from tests.reference_workflow.target_architecture import parse_graph, render_tables
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
SAMPLE_DATA,
WORKFLOW,
DeliverableKind,
dependencies_for,
sample_submission,
)
K = DeliverableKind
GRAPH = json.loads(SAMPLE_DATA["target-architecture.json"])
UPSTREAM = {
"requirements": {
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
},
"technical_decomposition": {
"attachments": {"components.json": SAMPLE_DATA["components.json"]}
},
"product_selection": {
"attachments": {"products.json": SAMPLE_DATA["products.json"]}
},
"design_recommendations": {
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
},
"target_architecture": {
"attachments": {
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
}
},
}
def tools_of(kind, working, upstream=UPSTREAM, extra=()):
return {
tool.name: tool
for tool in specialist_tools(
WORKFLOW,
kind,
extra,
allow_cloud_read=False,
upstream=upstream,
working=working,
)
}
# --- the Target architecture is written once -------------------------------------------
def test_tables_are_rendered_from_the_graph():
tables = render_tables(parse_graph(SAMPLE_DATA["target-architecture.json"]))
assert "## Elements" in tables and "## Relationships" in tables
assert (
"| E-1 | Ingress service | Example Run | example_run_service | C-1 |" in tables
)
assert "| R-1 | Ingress service | runs as | Ingress identity |" in tables
assert "| K-1 | E-1 | The service runs at least two instances. | NFR-1 |" in tables
def test_a_table_cell_cannot_break_its_row():
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
graph["checks"][0]["statement"] = "a | b\nc"
assert "| a \\| b c |" in render_tables(parse_graph(json.dumps(graph)))
async def test_the_engine_writes_the_tables_and_the_diagram_on_submission():
tools = tools_of(K.TARGET_ARCHITECTURE, {})
result = await tools["submit_deliverable"].handler(
content="## Design rationale\nOne service, one identity.\n",
attachments={
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
},
)
assert isinstance(result.outcome, Submitted)
assert result.outcome.content.startswith("## Design rationale")
assert "## Checks" in result.outcome.content
assert result.outcome.attachments["architecture.mmd"].startswith("flowchart")
async def test_the_graph_is_built_a_few_items_at_a_time_and_attached_on_submission():
working: dict = {}
tools = tools_of(K.TARGET_ARCHITECTURE, working)
await tools["graph_put"].handler(elements=GRAPH["elements"])
await tools["graph_put"].handler(relationships=GRAPH["relationships"])
put = await tools["graph_put"].handler(checks=GRAPH["checks"])
assert put.message == (
"Added 2, replaced 0. The graph has 2 Elements, 1 Relationships and 2 Checks."
)
result = await tools["submit_deliverable"].handler(
content="## Design rationale\nReasons.\n"
)
assert isinstance(result.outcome, Submitted)
submitted = json.loads(result.outcome.attachments["target-architecture.json"])
assert [item["id"] for item in submitted["checks"]] == ["K-1", "K-2"]
async def test_a_revision_amends_the_graph_that_was_submitted():
working: dict = {}
tools = tools_of(K.TARGET_ARCHITECTURE, working)
await tools["submit_deliverable"].handler(
content="## Design rationale\nReasons.\n",
attachments={
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
},
)
# A later attempt, with the working memory the Execution kept.
tools = tools_of(K.TARGET_ARCHITECTURE, working)
changed = {**GRAPH["checks"][0], "statement": "min_instance_count is at least 2"}
put = await tools["graph_put"].handler(checks=[changed])
read = await tools["graph_read"].handler()
assert put.message.startswith("Added 0, replaced 1.")
assert "min_instance_count is at least 2" in read.message
assert len(json.loads(read.message)["checks"]) == 2
async def test_graph_items_can_be_removed_and_an_item_without_an_id_is_refused():
working: dict = {}
tools = {tool.name: tool for tool in graph_tools(working, GRAPH_SHAPE)}
await tools["graph_put"].handler(
elements=GRAPH["elements"],
not_applicable=[{"recommendation": "DR-9", "reason": "Single region."}],
)
removed = await tools["graph_remove"].handler(ids=["E-2", "DR-9"])
refused = await tools["graph_put"].handler(checks=[{"statement": "no id"}])
assert removed.message.startswith("Removed 2.")
assert refused.message.startswith("Refused") and "checks[0]" in refused.message
assert [e["id"] for e in working["graph"]["elements"]] == ["E-1"]
# --- files are written once and changed in place ----------------------------------------
async def test_a_file_is_written_once_and_changed_in_place():
working: dict = {}
tools = {tool.name: tool for tool in file_tools(working)}
await tools["write_file"].handler(
path="terraform/main.tf", content="a = 1\nb = 1\n"
)
edited = await tools["edit_file"].handler(
path="terraform/main.tf", old="b = 1", new="b = 2"
)
listing = await tools["read_file"].handler()
assert edited.message == "Edited terraform/main.tf."
assert working["files"] == {"terraform/main.tf": "a = 1\nb = 2\n"}
assert "- terraform/main.tf (2 lines)" in listing.message
async def test_an_edit_that_does_not_match_exactly_once_changes_nothing():
working: dict = {"files": {"main.tf": "x = 1\nx = 1\n"}}
tools = {tool.name: tool for tool in file_tools(working)}
twice = await tools["edit_file"].handler(path="main.tf", old="x = 1", new="x = 2")
never = await tools["edit_file"].handler(path="main.tf", old="y", new="z")
absent = await tools["edit_file"].handler(path="other.tf", old="x", new="y")
unsafe = await tools["write_file"].handler(path="../main.tf", content="x")
assert "it matches 2" in twice.message and "it matches 0" in never.message
assert absent.message.startswith("Refused") and unsafe.message.startswith("Refused")
assert working["files"] == {"main.tf": "x = 1\nx = 1\n"}
async def test_the_working_copy_is_what_is_submitted():
working: dict = {}
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
for path in ("terraform/main.tf", "terraform/variables.tf"):
await tools["write_file"].handler(path=path, content=f"# {path}\n")
await tools["write_file"].handler(
path="implementation.json", content=SAMPLE_DATA["implementation.json"]
)
content = sample_submission(K.DEPLOYMENT_GUIDANCE).args["content"]
result = await tools["submit_deliverable"].handler(content=content)
assert isinstance(result.outcome, Submitted)
assert set(result.outcome.attachments) == {
"terraform/main.tf",
"terraform/variables.tf",
"implementation.json",
}
async def test_a_file_passed_on_submission_replaces_its_working_copy():
working: dict = {"files": {"terraform/main.tf": "old\n"}}
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
call = sample_submission(K.DEPLOYMENT_GUIDANCE)
result = await tools["submit_deliverable"].handler(**call.args)
assert isinstance(result.outcome, Submitted)
assert result.outcome.attachments["terraform/main.tf"] != "old\n"
assert working["files"]["terraform/main.tf"] != "old\n"
def test_only_a_specialist_that_owns_files_can_write_them():
assert "write_file" in tools_of(K.DEPLOYMENT_GUIDANCE, {})
assert "write_file" not in tools_of(K.VALIDATION_RESULTS, {})
assert "graph_put" in tools_of(K.TARGET_ARCHITECTURE, {})
assert "graph_put" not in tools_of(K.DEPLOYMENT_GUIDANCE, {})
# --- Terraform runs on the working copy -------------------------------------------------
class Runner(TerraformRunner):
def __init__(self) -> None:
super().__init__()
self.validated: list[dict[str, str]] = []
async def validate(self, files):
self.validated.append(dict(files))
return ToolResult("terraform validate passed:\n")
async def formatted(self, files):
return {name: content.strip() + "\n" for name, content in files.items()}
async def test_terraform_checks_take_no_files_when_there_is_a_working_copy():
runner = Runner()
files = {"terraform/main.tf": "x = 1\n", "implementation.json": "[]"}
(validate,) = local_tools(["terraform_validate"], runner, files)
await validate.handler()
assert validate.parameters["required"] == []
assert "pass no files" in validate.description
# Only the Terraform files are checked.
assert runner.validated == [{"terraform/main.tf": "x = 1\n"}]
async def test_formatting_changes_the_working_copy_and_returns_only_names():
files = {"terraform/main.tf": " x = 1 \n"}
(fmt,) = local_tools(["terraform_fmt"], Runner(), files)
result = await fmt.handler()
assert files == {"terraform/main.tf": "x = 1\n"}
assert result.message == (
"terraform fmt reformatted 1 file(s) in your working copy: terraform/main.tf."
)
async def test_files_passed_anyway_are_written_to_the_working_copy_first():
runner = Runner()
files: dict[str, str] = {}
(validate,) = local_tools(["terraform_validate"], runner, files)
await validate.handler(files=[{"path": "terraform/main.tf", "content": "y"}])
assert files == {"terraform/main.tf": "y"}
assert runner.validated == [files]
def roles_with(tools_by_role: dict[str, tuple[str, ...]]) -> dict[str, RoleSpec]:
return {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(
spec.specialist,
"Produce.",
budget=15,
tools=tools_by_role.get(spec.specialist, ()),
)
for spec in DEPENDENCY_GRAPH.values()
},
}
async def test_the_validator_checks_the_submitted_code_without_being_sent_it(tmp_path):
runner = Runner()
roles = roles_with({"validator": ("terraform_validate",)})
turns = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
turns["validator"] = [
[Call("terraform_validate", {}), sample_submission(K.VALIDATION_RESULTS)]
]
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
tool_factory=make_tool_factory(
WORKFLOW,
runtime,
roles,
local_tools=lambda working_copy: local_tools(
["terraform_validate"], runner, working_copy
),
),
)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert runner.validated == [
{
"terraform/main.tf": "# terraform/main.tf\n",
"terraform/variables.tf": "# terraform/variables.tf\n",
}
]
# The code is the code author's: validation attaches its verdicts and nothing else.
validation = result.state["deliverables"]["validation_results"]
assert set(validation["attachments"]) == {"verdicts.json"}
# --- each Specialist reads what it works from -------------------------------------------
def submitted() -> dict[str, dict]:
return {
str(kind): {
"content": f"# {kind}\nProse.\n",
"attachments": {
name: SAMPLE_DATA.get(name, f"# {name}\n") for name in spec.attachments
},
}
for kind, spec in DEPENDENCY_GRAPH.items()
}
def test_the_code_is_written_from_the_graph_and_the_ids_alone():
given = dependencies_for(K.DEPLOYMENT_GUIDANCE, submitted())
assert given["target_architecture"] == {
"attachments": {
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
}
}
assert all("content" not in deliverable for deliverable in given.values())
assert set(given["requirements"]["attachments"]) == {"requirements.json"}
def test_a_direct_dependency_is_read_whole_and_the_rest_as_data():
given = dependencies_for(K.TARGET_ARCHITECTURE, submitted())
assert "content" in given["product_selection"]
assert "content" in given["design_recommendations"]
assert "content" not in given["requirements"]
assert "content" not in given["technical_decomposition"]
def test_the_validator_reads_the_code_whole_and_the_graph_as_data():
given = dependencies_for(K.VALIDATION_RESULTS, submitted())
assert "terraform/main.tf" in given["deployment_guidance"]["attachments"]
assert "content" not in given["target_architecture"]
# A Deliverable with no data file has nothing to trim to, so it comes whole.
assert "content" in given["architecture_description"]
def test_the_solution_guide_is_assembled_from_every_deliverable_whole():
given = dependencies_for(K.SOLUTION_GUIDE, submitted())
# Every Deliverable but the guide itself and the plan produced beside it.
assert len(given) == len(DEPENDENCY_GRAPH) - 2
assert all("content" in deliverable for deliverable in given.values())
async def test_a_specialist_is_told_what_its_working_copy_already_holds(tmp_path):
turns = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
failing = sample_submission(K.VALIDATION_RESULTS)
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
verdicts["checks"][0].update(verdict="fail", evidence="min instances is 1")
turns["validator"] = [
[
Call(
failing.tool,
{
**failing.args,
"attachments": {"verdicts.json": json.dumps(verdicts)},
},
)
],
[sample_submission(K.VALIDATION_RESULTS)],
]
turns["iac-author"] = [[sample_submission(K.DEPLOYMENT_GUIDANCE)]] * 2
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=roles_with({}),
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
await engine.start("Ingest webhooks", pre_approved=True)
first, second = (run for run in runtime.runs if run.role == "iac-author")
assert first.task.context["working_copy"] is None
assert second.task.context["working_copy"] == {
"files": ["implementation.json", "terraform/main.tf", "terraform/variables.tf"]
}
async def test_a_deliverable_given_as_data_can_be_read_whole_on_demand():
library = {
"requirements": {
"content": "# requirements\nProse.\n",
"attachments": {"requirements.json": "[]"},
}
}
tools = {
tool.name: tool
for tool in specialist_tools(
WORKFLOW, K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False, library=library
)
}
read = await tools["read_deliverable"].handler(kind="requirements")
refused = await tools["read_deliverable"].handler(kind="solution_guide")
assert (
read.message.startswith("# requirements")
and "requirements.json" in read.message
)
assert refused.message.startswith("Refused")
# With nothing trimmed there is nothing to read back.
assert "read_deliverable" not in tools_of(K.REQUIREMENTS, {})
async def test_the_engine_offers_the_trimmed_deliverables_for_reading(tmp_path):
turns = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=roles_with({}),
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
await engine.start("Ingest webhooks", pre_approved=True)
author = next(run for run in runtime.runs if run.role == "iac-author")
packager = next(run for run in runtime.runs if run.role == "packager")
assert "read_deliverable" in author.tools
# The author is shown the graph alone; the rest it can read when it needs to.
assert list(author.task.context["dependencies"]) == ["target_architecture"]
assert "content" not in author.task.context["dependencies"]["target_architecture"]
assert "read_deliverable" not in packager.tools