feat: working orhcestration engine
This commit is contained in:
0
tests/execution/__init__.py
Normal file
0
tests/execution/__init__.py
Normal file
102
tests/execution/test_assessment.py
Normal file
102
tests/execution/test_assessment.py
Normal file
@@ -0,0 +1,102 @@
|
||||
"""Every Execution is judged as it ends (iteration 5 step 19)."""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import FakeAgentRuntime, FakeDecider
|
||||
from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from tests.reference_workflow.assessment import QUALITY_RUBRIC, assess
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
WORKFLOW,
|
||||
sample_submission,
|
||||
)
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
ADVICE = {"design_recommendations": {"content": "Use two zones for the database."}}
|
||||
|
||||
|
||||
def graded(level: float) -> FakeDecider:
|
||||
return FakeDecider({key: Answer(level, 0.9) for key in QUALITY_RUBRIC})
|
||||
|
||||
|
||||
async def test_the_rubric_is_scored_from_nought_to_one_with_its_mean():
|
||||
decider = graded(3.0)
|
||||
decider.answers["specific"] = Answer(1.5, 0.9)
|
||||
|
||||
assessment = await assess(decider, ADVICE)
|
||||
|
||||
assert assessment == {
|
||||
"recommendations_quality": {
|
||||
"specific": 0.5,
|
||||
"actionable": 1.0,
|
||||
"trade_offs": 1.0,
|
||||
"traceable": 1.0,
|
||||
},
|
||||
"mean": 0.88,
|
||||
"meets_bar": True,
|
||||
}
|
||||
document, _, _ = decider.asked[0]
|
||||
assert document == "Use two zones for the database."
|
||||
|
||||
|
||||
async def test_a_weak_design_is_reported_as_below_the_bar():
|
||||
assessment = await assess(graded(1.0), ADVICE)
|
||||
|
||||
assert assessment["mean"] == 0.33 and assessment["meets_bar"] is False
|
||||
|
||||
|
||||
async def test_nothing_to_judge_or_no_decision_gives_no_assessment():
|
||||
assert await assess(graded(3.0), {}) is None
|
||||
assert await assess(NoDecider(), ADVICE) is None
|
||||
|
||||
|
||||
def engine(tmp_path, decider, **options) -> ExecutionEngine:
|
||||
return ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=FakeAgentRuntime(
|
||||
{
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
),
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
decider=decider,
|
||||
**options,
|
||||
)
|
||||
|
||||
|
||||
async def test_a_completed_execution_carries_its_assessment(tmp_path):
|
||||
result = await engine(tmp_path, graded(1.0), assess_at_end=True).start(
|
||||
"Ingest webhooks", pre_approved=True
|
||||
)
|
||||
|
||||
# A low score is reported and the Execution still completes.
|
||||
assert result.status == "completed"
|
||||
assert result.state["assessment"]["meets_bar"] is False
|
||||
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
|
||||
assert manifest["assessment"] == result.state["assessment"]
|
||||
|
||||
|
||||
async def test_an_execution_is_not_judged_unless_that_is_switched_on(tmp_path):
|
||||
decider = graded(3.0)
|
||||
|
||||
result = await engine(tmp_path, decider).start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.state["assessment"] is None and decider.asked == []
|
||||
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
|
||||
assert manifest["assessment"] is None
|
||||
772
tests/execution/test_change_requests.py
Normal file
772
tests/execution/test_change_requests.py
Normal file
@@ -0,0 +1,772 @@
|
||||
"""Change requests (ADR-0006): an Execution that starts from a Baseline.
|
||||
|
||||
A Baseline is a completed Execution of the same Customer case. A Change request is a
|
||||
new Execution that begins with the Baseline's Deliverables; the Baseline itself is
|
||||
never modified.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from execution_engine.adk.runtime import NoDecider, RoleSpec
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.baseline import BaselineError, load_baseline
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
SAMPLE_DATA,
|
||||
WORKFLOW,
|
||||
DeliverableKind,
|
||||
sample_submission,
|
||||
)
|
||||
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
|
||||
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
|
||||
|
||||
K = DeliverableKind
|
||||
CASE = "acme-webhooks"
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def once() -> dict:
|
||||
return {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
|
||||
|
||||
def revised(**changes) -> Call:
|
||||
"""The requirements of a Change request: the Baseline's, each marked, plus what
|
||||
`changes` adds or alters by id."""
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
for item in requirements:
|
||||
item["change"] = "unchanged"
|
||||
if item["id"] in changes:
|
||||
item.update(changes.pop(item["id"]))
|
||||
requirements += [{"id": key, **value} for key, value in changes.items()]
|
||||
call = sample_submission(K.REQUIREMENTS)
|
||||
return Call(
|
||||
call.tool,
|
||||
{
|
||||
**call.args,
|
||||
"attachments": {"requirements.json": json.dumps(requirements)},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
RETENTION = {
|
||||
"NFR-1": {
|
||||
"change": "changed",
|
||||
"text": "99.99% availability.",
|
||||
"reason": "The service is now customer-facing.",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def changed(**overrides) -> dict:
|
||||
"""Scripted turns for a Change request that changes NFR-1 and nothing else."""
|
||||
turns = once()
|
||||
turns["requirements-analyst"] = [[revised(**json.loads(json.dumps(RETENTION)))]]
|
||||
turns.update(overrides)
|
||||
return turns
|
||||
|
||||
|
||||
def engine_for(tmp_path: Path, turns: dict, **options) -> ExecutionEngine:
|
||||
return ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=FakeAgentRuntime(turns),
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
**options,
|
||||
)
|
||||
|
||||
|
||||
async def delivered(tmp_path: Path, case: str = CASE, execution_id: str = "v1"):
|
||||
"""A completed Execution on disk, to be a Baseline."""
|
||||
result = await engine_for(tmp_path, once()).start(
|
||||
"Ingest webhooks",
|
||||
customer_case=case,
|
||||
pre_approved=True,
|
||||
execution_id=execution_id,
|
||||
)
|
||||
assert result.status == "completed"
|
||||
return result
|
||||
|
||||
|
||||
def digest(folder: Path) -> dict[str, str]:
|
||||
return {
|
||||
str(path.relative_to(folder)): hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
for path in sorted(folder.rglob("*"))
|
||||
if path.is_file()
|
||||
}
|
||||
|
||||
|
||||
def history(tmp_path: Path, case: str = CASE) -> list[dict]:
|
||||
path = tmp_path / "deliverables" / "cases" / case / "intent" / "history.json"
|
||||
return json.loads(path.read_text())["intents"]
|
||||
|
||||
|
||||
# --- step 2: start against a Baseline --------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_baseline_is_read_from_its_files(tmp_path):
|
||||
await delivered(tmp_path)
|
||||
|
||||
baseline = load_baseline(tmp_path, CASE, "v1")
|
||||
|
||||
assert baseline["execution_id"] == "v1"
|
||||
assert baseline["workflow_request"] == "Ingest webhooks"
|
||||
assert set(baseline["deliverables"]) == {kind.value for kind in DEPENDENCY_GRAPH}
|
||||
graph = baseline["deliverables"]["target_architecture"]["attachments"]
|
||||
assert "target-architecture.json" in graph
|
||||
assert baseline["deliverables"]["requirements"]["content"].startswith("#")
|
||||
|
||||
|
||||
async def test_a_change_request_starts_with_its_baseline_and_is_recorded(tmp_path):
|
||||
await delivered(tmp_path)
|
||||
|
||||
result = await engine_for(tmp_path, changed()).start(
|
||||
"Retention goes from 30 days to 7 years.",
|
||||
customer_case=CASE,
|
||||
pre_approved=True,
|
||||
execution_id="v2",
|
||||
baseline="v1",
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.state["baseline"]["execution_id"] == "v1"
|
||||
first, second = history(tmp_path)
|
||||
assert first["baseline"] is None
|
||||
assert second["baseline"] == "v1" and second["execution_id"] == "v2"
|
||||
case = tmp_path / "deliverables" / "cases" / CASE
|
||||
manifest = json.loads((case / "executions" / "v2" / "execution.json").read_text())
|
||||
assert manifest["baseline"] == "v1"
|
||||
assert "- Changes: `v1`" in (case / "intent" / "002.md").read_text()
|
||||
|
||||
|
||||
async def test_the_baseline_is_never_modified(tmp_path):
|
||||
await delivered(tmp_path)
|
||||
folder = tmp_path / "deliverables" / "cases" / CASE / "executions" / "v1"
|
||||
before = digest(folder)
|
||||
|
||||
await engine_for(tmp_path, changed()).start(
|
||||
"Add a second region.",
|
||||
customer_case=CASE,
|
||||
pre_approved=True,
|
||||
execution_id="v2",
|
||||
baseline="v1",
|
||||
)
|
||||
|
||||
assert digest(folder) == before
|
||||
|
||||
|
||||
async def test_a_change_request_may_itself_be_a_baseline(tmp_path):
|
||||
await delivered(tmp_path)
|
||||
for version, previous in (("v2", "v1"), ("v3", "v2")):
|
||||
await engine_for(tmp_path, changed()).start(
|
||||
"Another change.",
|
||||
customer_case=CASE,
|
||||
pre_approved=True,
|
||||
execution_id=version,
|
||||
baseline=previous,
|
||||
)
|
||||
|
||||
assert [item["baseline"] for item in history(tmp_path)] == [None, "v1", "v2"]
|
||||
assert load_baseline(tmp_path, CASE, "v3")["baseline"] == "v2"
|
||||
|
||||
|
||||
async def test_without_a_baseline_an_execution_starts_clean(tmp_path):
|
||||
result = await delivered(tmp_path)
|
||||
|
||||
assert result.state["baseline"] is None
|
||||
manifest = json.loads(
|
||||
(
|
||||
tmp_path / "deliverables/cases" / CASE / "executions/v1/execution.json"
|
||||
).read_text()
|
||||
)
|
||||
assert manifest["baseline"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case, baseline, message",
|
||||
[
|
||||
(CASE, "nope", "is not an Execution of Customer case acme-webhooks"),
|
||||
("other-client", "v1", "belongs to another Customer case"),
|
||||
(None, "v1", "needs a customer_case"),
|
||||
(CASE, "../v1", "is not an Execution id"),
|
||||
],
|
||||
)
|
||||
async def test_a_baseline_that_cannot_be_one_refuses_the_start(
|
||||
tmp_path, case, baseline, message
|
||||
):
|
||||
await delivered(tmp_path)
|
||||
engine = engine_for(tmp_path, once())
|
||||
|
||||
with pytest.raises(BaselineError, match=message):
|
||||
await engine.start(
|
||||
"A change.", customer_case=case, execution_id="v2", baseline=baseline
|
||||
)
|
||||
|
||||
# Refused before anything ran or was recorded.
|
||||
assert [item["execution_id"] for item in history(tmp_path)] == ["v1"]
|
||||
assert not (tmp_path / "deliverables/cases/other-client").exists()
|
||||
|
||||
|
||||
async def test_an_execution_that_did_not_complete_cannot_be_a_baseline(tmp_path):
|
||||
from tests.execution.test_router import failing
|
||||
|
||||
turns = once()
|
||||
turns["validator"] = [[failing("K-1")]]
|
||||
engine = engine_for(tmp_path, turns, max_revisions=0)
|
||||
unfinished = await engine.start(
|
||||
"Ingest webhooks", customer_case=CASE, pre_approved=True, execution_id="v1"
|
||||
)
|
||||
assert unfinished.status == "completed_with_warnings"
|
||||
|
||||
with pytest.raises(BaselineError, match="did not complete"):
|
||||
load_baseline(tmp_path, CASE, "v1")
|
||||
|
||||
|
||||
# --- step 3: the requirements say what changed -------------------------------------------
|
||||
|
||||
|
||||
def requirements_check(revised_call: Call, baseline=None) -> tuple[str, ...]:
|
||||
from tests.reference_workflow.workflow import check_submission
|
||||
|
||||
baseline = baseline or {
|
||||
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
||||
}
|
||||
return check_submission(
|
||||
K.REQUIREMENTS,
|
||||
revised_call.args["content"],
|
||||
revised_call.args["attachments"],
|
||||
baseline=baseline,
|
||||
)
|
||||
|
||||
|
||||
def test_a_revised_requirement_list_that_accounts_for_every_change_is_accepted():
|
||||
call = revised(
|
||||
**{
|
||||
"NFR-1": RETENTION["NFR-1"],
|
||||
"NFR-2": {
|
||||
"kind": "non_functional",
|
||||
"text": "A second region.",
|
||||
"conditions": ["The service runs in two regions."],
|
||||
"change": "added",
|
||||
"reason": "Customers are now in two continents.",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert requirements_check(call) == ()
|
||||
|
||||
|
||||
def test_every_requirement_in_a_change_request_says_whether_it_changed():
|
||||
call = sample_submission(K.REQUIREMENTS) # no `change` on anything
|
||||
|
||||
missing = requirements_check(call)
|
||||
|
||||
assert "requirements.json: FR-1 needs a `change`" in missing[0]
|
||||
assert len(missing) == 2
|
||||
|
||||
|
||||
def test_a_change_needs_its_reason_and_an_unchanged_requirement_its_text():
|
||||
no_reason = revised(**{"NFR-1": {"change": "changed", "text": "99.99%."}})
|
||||
quietly = revised(**{"NFR-1": {"text": "99.99% availability."}})
|
||||
quietly_other = revised(
|
||||
**{"NFR-1": RETENTION["NFR-1"], "FR-1": {"text": "Accept events and files."}}
|
||||
)
|
||||
|
||||
assert requirements_check(no_reason) == (
|
||||
"requirements.json: NFR-1 is changed and needs a `reason`",
|
||||
)
|
||||
assert (
|
||||
"NFR-1 is marked unchanged, but its text differs"
|
||||
in (requirements_check(quietly)[0])
|
||||
)
|
||||
assert (
|
||||
"FR-1 is marked unchanged, but its text differs"
|
||||
in (requirements_check(quietly_other)[0])
|
||||
)
|
||||
|
||||
|
||||
def test_a_baseline_requirement_cannot_silently_disappear_or_be_reused():
|
||||
call = revised(**{"NFR-1": RETENTION["NFR-1"]})
|
||||
requirements = json.loads(call.args["attachments"]["requirements.json"])
|
||||
dropped = Call(
|
||||
call.tool,
|
||||
{
|
||||
**call.args,
|
||||
"attachments": {"requirements.json": json.dumps(requirements[1:])},
|
||||
},
|
||||
)
|
||||
reused = revised(
|
||||
**{
|
||||
"NFR-1": {
|
||||
"change": "added",
|
||||
"text": "Something else entirely.",
|
||||
"reason": "New.",
|
||||
}
|
||||
}
|
||||
)
|
||||
invented = revised(
|
||||
**{
|
||||
"NFR-1": RETENTION["NFR-1"],
|
||||
"NFR-9": {
|
||||
"kind": "non_functional",
|
||||
"text": "x",
|
||||
"conditions": ["x"],
|
||||
"change": "changed",
|
||||
"reason": "y",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert "Baseline requirement FR-1 is missing" in requirements_check(dropped)[0]
|
||||
assert (
|
||||
"NFR-1 is marked added, but the Baseline has that id"
|
||||
in (requirements_check(reused)[0])
|
||||
)
|
||||
assert (
|
||||
"NFR-9 is marked changed, but the Baseline has no such requirement"
|
||||
in (requirements_check(invented)[0])
|
||||
)
|
||||
|
||||
|
||||
def test_a_change_request_must_change_something():
|
||||
assert (
|
||||
"nothing is marked added, changed or removed"
|
||||
in (requirements_check(revised())[0])
|
||||
)
|
||||
|
||||
|
||||
def test_a_removed_requirement_stays_on_record_and_asks_nothing_downstream():
|
||||
from tests.reference_workflow.chain import read_list
|
||||
|
||||
call = revised(
|
||||
**{"NFR-1": {"change": "removed", "reason": "No longer a requirement."}}
|
||||
)
|
||||
upstream = {"requirements": {"attachments": call.args["attachments"]}}
|
||||
|
||||
assert requirements_check(call) == ()
|
||||
live = read_list(upstream, "requirements", "requirements.json")
|
||||
assert [item["id"] for item in live] == ["FR-1"]
|
||||
assert '"NFR-1"' in call.args["attachments"]["requirements.json"]
|
||||
|
||||
|
||||
# --- step 4: later Specialists revise from the Baseline ---------------------------------
|
||||
|
||||
|
||||
async def test_each_specialist_is_given_its_delivered_version_and_what_changed(
|
||||
tmp_path,
|
||||
):
|
||||
await delivered(tmp_path)
|
||||
runtime = FakeAgentRuntime(changed())
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
|
||||
await engine.start(
|
||||
"Availability goes up.",
|
||||
customer_case=CASE,
|
||||
pre_approved=True,
|
||||
execution_id="v2",
|
||||
baseline="v1",
|
||||
)
|
||||
|
||||
by_role = {run.role: run for run in runtime.runs}
|
||||
analyst = by_role["requirements-analyst"].task.context["baseline"]
|
||||
assert analyst["execution_id"] == "v1"
|
||||
assert analyst["original_request"] == "Ingest webhooks"
|
||||
assert "requirements.json" in analyst["deliverable"]["attachments"]
|
||||
assert analyst["changes"] is None # nothing has changed yet
|
||||
|
||||
architect = by_role["target-architect"].task.context
|
||||
assert architect["baseline"]["changes"]["requirements"] == [
|
||||
{
|
||||
"id": "NFR-1",
|
||||
"change": "changed",
|
||||
"reason": "The service is now customer-facing.",
|
||||
"text": "99.99% availability.",
|
||||
}
|
||||
]
|
||||
# Its working copy already holds the delivered graph.
|
||||
assert architect["baseline"]["deliverable"]["attachments"][
|
||||
"target-architecture.json"
|
||||
]
|
||||
assert "graph_read" in by_role["target-architect"].tools
|
||||
# The guide is assembled afresh: the Baseline's is not handed back.
|
||||
assert by_role["packager"].task.context["baseline"]["deliverable"] is None
|
||||
|
||||
|
||||
async def test_a_description_whose_design_did_not_change_shape_is_carried_over(
|
||||
tmp_path,
|
||||
):
|
||||
await delivered(tmp_path)
|
||||
runtime = FakeAgentRuntime(changed())
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
|
||||
result = await engine.start(
|
||||
"Availability goes up.",
|
||||
customer_case=CASE,
|
||||
pre_approved=True,
|
||||
execution_id="v2",
|
||||
baseline="v1",
|
||||
)
|
||||
|
||||
# The same Elements and Relationships as the Baseline: the describer is not run.
|
||||
assert "describer" not in {run.role for run in runtime.runs}
|
||||
assert result.state["deliverables"]["architecture_description"]["content"]
|
||||
|
||||
|
||||
async def test_a_code_authors_working_copy_starts_as_the_delivered_code(tmp_path):
|
||||
await delivered(tmp_path)
|
||||
seen: dict = {}
|
||||
|
||||
def author(task):
|
||||
seen["working_copy"] = task.context["working_copy"]
|
||||
return [sample_submission(K.DEPLOYMENT_GUIDANCE)]
|
||||
|
||||
engine = engine_for(tmp_path, changed(**{"iac-author": [author]}))
|
||||
|
||||
await engine.start(
|
||||
"Availability goes up.",
|
||||
customer_case=CASE,
|
||||
pre_approved=True,
|
||||
execution_id="v2",
|
||||
baseline="v1",
|
||||
)
|
||||
|
||||
# It is told nothing of a working copy on a first attempt; the tools hold it.
|
||||
assert seen["working_copy"] is None
|
||||
|
||||
|
||||
# --- what changed, worked out from the ids ------------------------------------------------
|
||||
|
||||
|
||||
def versions(graph_change=None, requirement_change=None) -> tuple[dict, dict]:
|
||||
"""A Baseline and a revised set of Deliverables, as data files."""
|
||||
|
||||
def deliverables() -> dict:
|
||||
return {
|
||||
kind: {"attachments": {name: SAMPLE_DATA[name]}}
|
||||
for kind, name in (
|
||||
("requirements", "requirements.json"),
|
||||
("technical_decomposition", "components.json"),
|
||||
("design_recommendations", "recommendations.json"),
|
||||
("target_architecture", "target-architecture.json"),
|
||||
)
|
||||
}
|
||||
|
||||
before, after = deliverables(), deliverables()
|
||||
change = requirement_change or RETENTION
|
||||
after["requirements"]["attachments"] = revised(
|
||||
**json.loads(json.dumps(change))
|
||||
).args["attachments"]
|
||||
if graph_change:
|
||||
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
graph_change(graph)
|
||||
after["target_architecture"]["attachments"] = {
|
||||
"target-architecture.json": json.dumps(graph)
|
||||
}
|
||||
return before, after
|
||||
|
||||
|
||||
def test_a_change_inside_the_reach_of_a_changed_requirement_is_requested():
|
||||
from tests.reference_workflow.changes import change_summary
|
||||
|
||||
def tighten(graph):
|
||||
graph["checks"][0]["statement"] = "The service runs at least three instances."
|
||||
|
||||
summary = change_summary(*versions(tighten))
|
||||
|
||||
assert [item["id"] for item in summary["requirements"]] == ["NFR-1"]
|
||||
assert summary["checks"] == {"added": [], "changed": ["K-1"], "removed": []}
|
||||
assert summary["elements"] == {"added": [], "changed": [], "removed": []}
|
||||
assert summary["unrequested"] == {}
|
||||
|
||||
|
||||
def test_a_change_no_changed_requirement_reaches_is_listed_as_unrequested():
|
||||
from tests.reference_workflow.changes import change_summary
|
||||
|
||||
def wander(graph):
|
||||
# K-2 comes from DR-1, which did not change, on E-2, which realises nothing
|
||||
# the changed requirement touches. NFR-1 reaches K-1 and E-1 only.
|
||||
graph["checks"][1]["statement"] = "The identity has no keys."
|
||||
|
||||
change = {
|
||||
"NFR-1": {"change": "changed", "text": "99.99%.", "reason": "Customer-facing."}
|
||||
}
|
||||
before, after = versions(wander, change)
|
||||
# NFR-1 is served by C-1, which E-1 realises; cut that link to isolate E-2.
|
||||
components = [{"id": "C-1", "name": "Ingress", "requirements": ["FR-1"]}]
|
||||
for version in (before, after):
|
||||
version["technical_decomposition"]["attachments"] = {
|
||||
"components.json": json.dumps(components)
|
||||
}
|
||||
|
||||
summary = change_summary(before, after)
|
||||
|
||||
assert summary["checks"]["changed"] == ["K-2"]
|
||||
assert summary["unrequested"] == {"checks": ["K-2"]}
|
||||
|
||||
|
||||
def test_added_and_removed_items_are_found_by_id():
|
||||
from tests.reference_workflow.changes import change_summary
|
||||
|
||||
def grow(graph):
|
||||
graph["elements"].append(
|
||||
{
|
||||
"id": "E-3",
|
||||
"name": "Second region service",
|
||||
"component": "C-1",
|
||||
"product": "Example Run",
|
||||
"resource_type": "example_run_service",
|
||||
}
|
||||
)
|
||||
graph["checks"] = [graph["checks"][0]] + [
|
||||
{"id": "K-3", "applies_to": "E-3", "statement": "x", "source": "NFR-1"}
|
||||
]
|
||||
|
||||
summary = change_summary(*versions(grow))
|
||||
|
||||
assert summary["elements"]["added"] == ["E-3"]
|
||||
assert summary["checks"] == {"added": ["K-3"], "changed": [], "removed": ["K-2"]}
|
||||
# K-2 came from DR-1 on E-2: nothing that changed reaches it.
|
||||
assert summary["unrequested"] == {"checks": ["K-2"]}
|
||||
|
||||
|
||||
# --- step 5: Approvals show the change --------------------------------------------------
|
||||
|
||||
|
||||
async def test_an_approval_in_a_change_request_shows_what_that_phase_changes(
|
||||
tmp_path,
|
||||
):
|
||||
from tests.reference_workflow.presentation import describe_prompt
|
||||
|
||||
await delivered(tmp_path)
|
||||
engine = engine_for(tmp_path, changed())
|
||||
|
||||
first = await engine.start(
|
||||
"Availability goes up.", customer_case=CASE, execution_id="v2", baseline="v1"
|
||||
)
|
||||
|
||||
approval = first.prompt["approval"]
|
||||
assert approval["phase"] == "requirements_discovery"
|
||||
assert approval["baseline"] == "v1"
|
||||
assert [item["id"] for item in approval["changes"]["requirements"]] == ["NFR-1"]
|
||||
text = describe_prompt(first.prompt)
|
||||
assert "This changes the architecture delivered in v1." in text
|
||||
assert (
|
||||
"- Requirement NFR-1 changed: 99.99% availability. (The service is now "
|
||||
"customer-facing.)"
|
||||
) in text
|
||||
|
||||
second = await engine.resume(
|
||||
"v2", {"approval": {"phase": "requirements_discovery", "decision": "approve"}}
|
||||
)
|
||||
|
||||
# The design did not change, and the Approval says so.
|
||||
design = second.prompt["approval"]
|
||||
assert design["phase"] == "solution_architecture" and design["changes"] == {}
|
||||
assert "Nothing in this Phase differs from it." in describe_prompt(second.prompt)
|
||||
|
||||
|
||||
async def test_an_ordinary_approval_says_nothing_of_changes(tmp_path):
|
||||
first = await engine_for(tmp_path, once()).start("Ingest webhooks")
|
||||
|
||||
assert "changes" not in first.prompt["approval"]
|
||||
|
||||
|
||||
# --- step 6: the change summary and the line of versions ----------------------------------
|
||||
|
||||
|
||||
async def test_a_change_request_writes_changes_json_and_a_full_set_of_files(tmp_path):
|
||||
from tests.reference_workflow.changes import change_summary
|
||||
|
||||
await delivered(tmp_path)
|
||||
|
||||
result = await engine_for(tmp_path, changed()).start(
|
||||
"Availability goes up.",
|
||||
customer_case=CASE,
|
||||
pre_approved=True,
|
||||
execution_id="v2",
|
||||
baseline="v1",
|
||||
)
|
||||
|
||||
case = tmp_path / "deliverables" / "cases" / CASE / "executions"
|
||||
written = json.loads((case / "v2" / "changes.json").read_text())
|
||||
assert [item["id"] for item in written["requirements"]] == ["NFR-1"]
|
||||
assert written["unrequested"] == {}
|
||||
# It matches a comparison of the two Executions' data files.
|
||||
assert written == change_summary(
|
||||
load_baseline(tmp_path, CASE, "v1")["deliverables"],
|
||||
result.state["deliverables"],
|
||||
)
|
||||
# The Change request stands on its own: every Deliverable is written in full.
|
||||
manifest = json.loads((case / "v2" / "execution.json").read_text())
|
||||
assert manifest["changes"] == "changes.json"
|
||||
assert set(manifest["deliverables"]) == {kind.value for kind in DEPENDENCY_GRAPH}
|
||||
assert (case / "v2" / "solution-architecture-guide.md").is_file()
|
||||
assert not (case / "v1" / "changes.json").exists()
|
||||
|
||||
|
||||
# --- only the pillars a change touches are consulted again ------------------------------
|
||||
|
||||
|
||||
async def test_a_change_request_starts_from_the_baselines_pillar_advice(tmp_path):
|
||||
"""The design advisor is told which pillars the change affects, and has the
|
||||
Baseline's advice for the rest."""
|
||||
from execution_engine.adk.fake_runtime import FakeDecider
|
||||
from execution_engine.adk.runtime import Answer
|
||||
from tests.reference_workflow.pillars import ADVICE_KEY
|
||||
|
||||
await delivered(tmp_path)
|
||||
# A Baseline whose design advisor consulted two pillars.
|
||||
folder = tmp_path / "deliverables" / "cases" / CASE / "executions" / "v1"
|
||||
advice = {"security": "Use a dedicated identity.", "reliability": "Two zones."}
|
||||
(folder / "working.json").write_text(
|
||||
json.dumps({"design_recommendations": {ADVICE_KEY: advice}})
|
||||
)
|
||||
seen: dict = {}
|
||||
|
||||
def advisor(task):
|
||||
seen["pillars_to_consult"] = task.context["pillars_to_consult"]
|
||||
return [sample_submission(K.DESIGN_RECOMMENDATIONS)]
|
||||
|
||||
decider = FakeDecider(
|
||||
{"security": Answer(0.3, 0.8), "reliability": Answer(1.9, 0.9)}
|
||||
)
|
||||
runtime = FakeAgentRuntime(changed(**{"design-advisor": [advisor]}))
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
decider=decider,
|
||||
)
|
||||
|
||||
result = await engine.start(
|
||||
"Availability goes up.",
|
||||
customer_case=CASE,
|
||||
pre_approved=True,
|
||||
execution_id="v2",
|
||||
baseline="v1",
|
||||
)
|
||||
|
||||
assert seen["pillars_to_consult"] == ["reliability"]
|
||||
state, _, _ = decider.asked[0]
|
||||
assert "Requirement NFR-1 changed: 99.99% availability." in state
|
||||
# The advice is carried into the new Execution, and written for the next change.
|
||||
assert result.state["working"]["design_recommendations"][ADVICE_KEY] == advice
|
||||
assert json.loads((folder.parent / "v2" / "working.json").read_text()) == {
|
||||
"design_recommendations": {ADVICE_KEY: advice}
|
||||
}
|
||||
|
||||
|
||||
async def test_a_first_consultation_is_given_no_hint(tmp_path):
|
||||
seen: dict = {}
|
||||
|
||||
def advisor(task):
|
||||
seen["pillars_to_consult"] = task.context["pillars_to_consult"]
|
||||
return [sample_submission(K.DESIGN_RECOMMENDATIONS)]
|
||||
|
||||
from execution_engine.adk.fake_runtime import FakeDecider
|
||||
|
||||
decider = FakeDecider({})
|
||||
turns = once()
|
||||
turns["design-advisor"] = [advisor]
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=FakeAgentRuntime(turns),
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
decider=decider,
|
||||
)
|
||||
|
||||
await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert seen["pillars_to_consult"] is None and decider.asked == []
|
||||
|
||||
|
||||
# --- changed, or only reworded? ---------------------------------------------------------
|
||||
|
||||
|
||||
async def test_an_item_that_was_only_reworded_is_listed_apart_from_real_changes():
|
||||
from execution_engine.adk.fake_runtime import FakeDecider
|
||||
from execution_engine.adk.runtime import Answer
|
||||
from tests.reference_workflow.changes import change_summary
|
||||
from tests.reference_workflow.meaning import settle_meaning
|
||||
|
||||
def revise(graph):
|
||||
graph["checks"][0]["statement"] = "The service runs at least three instances."
|
||||
graph["checks"][1]["statement"] = "This service alone uses the identity."
|
||||
|
||||
before, after = versions(revise)
|
||||
summary = change_summary(before, after)
|
||||
assert summary["checks"]["changed"] == ["K-1", "K-2"]
|
||||
decider = FakeDecider({"K-1": Answer(0.95, 0.9), "K-2": Answer(0.1, 0.8)})
|
||||
|
||||
settled = await settle_meaning(decider, summary, before, after, scope="v2")
|
||||
|
||||
assert settled["checks"]["changed"] == ["K-1"]
|
||||
assert settled["checks"]["reworded"] == ["K-2"]
|
||||
assert settled["meaning"] == {"K-1": 0.95, "K-2": 0.1}
|
||||
# K-2 was outside the reach of what changed; reworded, it is no longer unrequested.
|
||||
assert settled["unrequested"] == {}
|
||||
_, questions, scope = decider.asked[0]
|
||||
assert scope == "v2"
|
||||
assert "The identity is used by this service only." in questions["K-2"].instructions
|
||||
assert "This service alone uses the identity." in questions["K-2"].instructions
|
||||
# The summary given in is not altered.
|
||||
assert summary["checks"]["changed"] == ["K-1", "K-2"]
|
||||
|
||||
|
||||
async def test_without_a_decision_everything_that_differs_stays_changed():
|
||||
from tests.reference_workflow.changes import change_summary
|
||||
from tests.reference_workflow.meaning import settle_meaning
|
||||
|
||||
def revise(graph):
|
||||
graph["checks"][1]["statement"] = "This service alone uses the identity."
|
||||
|
||||
before, after = versions(revise)
|
||||
summary = change_summary(before, after)
|
||||
|
||||
settled = await settle_meaning(NoDecider(), summary, before, after)
|
||||
|
||||
assert settled["checks"]["changed"] == ["K-2"]
|
||||
assert "reworded" not in settled["checks"] and "meaning" not in settled
|
||||
446
tests/execution/test_coverage_review.py
Normal file
446
tests/execution/test_coverage_review.py
Normal file
@@ -0,0 +1,446 @@
|
||||
"""Two savings on a Revision of the design.
|
||||
|
||||
A coverage review at submission finds what validation would otherwise find later,
|
||||
at the cost of a Revision. And a Deliverable written from the shape of the design is
|
||||
not written again when a Revision only changes its Checks.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime, FakeDecider
|
||||
from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec, Submitted
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from execution_engine.tools.execution import specialist_tools
|
||||
from tests.reference_workflow.coverage import CoverageReview
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
SAMPLE_DATA,
|
||||
WORKFLOW,
|
||||
DeliverableKind,
|
||||
sample_submission,
|
||||
)
|
||||
|
||||
K = DeliverableKind
|
||||
GRAPH = {"target-architecture.json": SAMPLE_DATA["target-architecture.json"]}
|
||||
UPSTREAM = {
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
||||
},
|
||||
"technical_decomposition": {
|
||||
"attachments": {"components.json": SAMPLE_DATA["components.json"]}
|
||||
},
|
||||
"product_selection": {
|
||||
"attachments": {"products.json": SAMPLE_DATA["products.json"]}
|
||||
},
|
||||
"design_recommendations": {
|
||||
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
|
||||
},
|
||||
}
|
||||
CONTENT = "## Design rationale\nReasons.\n"
|
||||
|
||||
|
||||
# --- the review ------------------------------------------------------------------------
|
||||
|
||||
|
||||
def choice(option: str, probability: float) -> Answer:
|
||||
return Answer(option, probability, {option: probability})
|
||||
|
||||
|
||||
async def test_the_review_asks_which_check_verifies_each_condition():
|
||||
decider = FakeDecider({"NFR-1#1": choice("K-1", 0.9), "DR-1#1": choice("K-2", 0.9)})
|
||||
|
||||
concerns = await CoverageReview(decider)(
|
||||
K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1"
|
||||
)
|
||||
|
||||
state, questions, scope = decider.asked[0]
|
||||
assert concerns == ()
|
||||
assert scope == "exec-1"
|
||||
assert "E-1 Ingress service (Example Run, example_run_service)" in state
|
||||
# Constraints only: a functional requirement is carried by its component.
|
||||
assert set(questions) == {"NFR-1#1", "DR-1#1"}
|
||||
question = questions["NFR-1#1"]
|
||||
assert question.kind == "choice" and question.options == ("K-1", "none")
|
||||
assert 'Condition: "The service stays available when one instance fails."' in (
|
||||
question.instructions
|
||||
)
|
||||
assert "- K-1 on Ingress service: The service runs at least two instances." in (
|
||||
question.instructions
|
||||
)
|
||||
|
||||
|
||||
async def test_a_condition_no_check_verifies_is_raised_with_the_checks_it_has():
|
||||
decider = FakeDecider({"NFR-1#1": choice("none", 0.9), "DR-1#1": choice("K-2", 1)})
|
||||
|
||||
concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM)
|
||||
|
||||
condition = "The service stays available when one instance fails."
|
||||
assert concerns == (f'NFR-1: no Check verifies "{condition}" (its Checks: K-1)',)
|
||||
|
||||
|
||||
async def test_an_uncertain_none_is_not_raised():
|
||||
decider = FakeDecider({"NFR-1#1": choice("none", 0.45), "DR-1#1": choice("K-2", 1)})
|
||||
|
||||
review = CoverageReview(decider, none_at=0.6)
|
||||
|
||||
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
|
||||
|
||||
|
||||
async def test_each_condition_of_a_requirement_is_asked_about_separately():
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
requirements[1]["conditions"] = [
|
||||
"Survives one instance failing.",
|
||||
"Survives a zone failing.",
|
||||
]
|
||||
upstream = {
|
||||
**UPSTREAM,
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": json.dumps(requirements)}
|
||||
},
|
||||
}
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"NFR-1#1": choice("K-1", 0.9),
|
||||
"NFR-1#2": choice("none", 0.8),
|
||||
"DR-1#1": choice("K-2", 1),
|
||||
}
|
||||
)
|
||||
|
||||
concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream)
|
||||
|
||||
assert len(concerns) == 1 and "Survives a zone failing." in concerns[0]
|
||||
|
||||
|
||||
async def test_an_item_without_conditions_is_read_as_one_condition_its_text():
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
del requirements[1]["conditions"]
|
||||
upstream = {
|
||||
**UPSTREAM,
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": json.dumps(requirements)}
|
||||
},
|
||||
}
|
||||
decider = FakeDecider({"NFR-1#1": choice("K-1", 1), "DR-1#1": choice("K-2", 1)})
|
||||
|
||||
await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream)
|
||||
|
||||
assert 'Condition: "99.9% availability."' in (
|
||||
decider.asked[0][1]["NFR-1#1"].instructions
|
||||
)
|
||||
|
||||
|
||||
async def test_no_decision_and_other_deliverables_raise_nothing():
|
||||
review = CoverageReview(NoDecider())
|
||||
decider = FakeDecider({})
|
||||
|
||||
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
|
||||
assert await CoverageReview(decider)(K.REQUIREMENTS, {}, UPSTREAM) == ()
|
||||
assert await CoverageReview(decider)(K.TARGET_ARCHITECTURE, {}, UPSTREAM) == ()
|
||||
assert decider.asked == []
|
||||
|
||||
|
||||
async def test_a_recommendation_marked_not_applicable_is_not_reviewed():
|
||||
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
graph["checks"][1]["source"] = "NFR-1"
|
||||
graph["not_applicable"] = [{"recommendation": "DR-1", "reason": "No identity."}]
|
||||
decider = FakeDecider({"NFR-1#1": choice("K-1", 1)})
|
||||
|
||||
await CoverageReview(decider)(
|
||||
K.TARGET_ARCHITECTURE,
|
||||
{"target-architecture.json": json.dumps(graph)},
|
||||
UPSTREAM,
|
||||
)
|
||||
|
||||
assert set(decider.asked[0][1]) == {"NFR-1#1"}
|
||||
|
||||
|
||||
# --- at submission ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def submit_with(review, working=None):
|
||||
(submit,) = (
|
||||
tool
|
||||
for tool in specialist_tools(
|
||||
WORKFLOW,
|
||||
K.TARGET_ARCHITECTURE,
|
||||
(),
|
||||
allow_cloud_read=False,
|
||||
upstream=UPSTREAM,
|
||||
working={} if working is None else working,
|
||||
review=review,
|
||||
)
|
||||
if tool.name == "submit_deliverable"
|
||||
)
|
||||
return submit.handler
|
||||
|
||||
|
||||
async def test_what_the_review_raises_is_returned_once_and_then_accepted():
|
||||
calls = []
|
||||
|
||||
async def review(attachments):
|
||||
calls.append(attachments)
|
||||
return ['NFR-1: no Check verifies "Two instances." (its Checks: K-1)']
|
||||
|
||||
working: dict = {}
|
||||
submit = submit_with(review, working)
|
||||
|
||||
first = await submit(content=CONTENT, attachments=dict(GRAPH))
|
||||
second = await submit(content=CONTENT)
|
||||
|
||||
assert first.outcome is None
|
||||
assert first.message.startswith("Not submitted. A review found")
|
||||
assert "NFR-1: no Check verifies" in first.message
|
||||
assert "runs once" in first.message
|
||||
# The graph it sent is kept, so it amends that and does not send it again.
|
||||
assert isinstance(second.outcome, Submitted)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
async def test_a_structural_fault_is_reported_before_any_review():
|
||||
async def review(attachments):
|
||||
raise AssertionError("not reached")
|
||||
|
||||
result = await submit_with(review)(content=CONTENT, attachments={})
|
||||
|
||||
assert "attachment: target-architecture.json" in result.message
|
||||
|
||||
|
||||
async def test_a_review_that_fails_or_raises_nothing_never_stops_a_submission():
|
||||
async def broken(attachments):
|
||||
raise RuntimeError("service down")
|
||||
|
||||
async def quiet(attachments):
|
||||
return ()
|
||||
|
||||
for review in (broken, quiet):
|
||||
result = await submit_with(review)(content=CONTENT, attachments=dict(GRAPH))
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
|
||||
|
||||
# --- a description stands while the shape of the design does ---------------------------
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def graph_with(**changes) -> Call:
|
||||
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
for key, change in changes.items():
|
||||
graph[key] = change(graph[key])
|
||||
return Call(
|
||||
"submit_deliverable",
|
||||
{
|
||||
"content": CONTENT,
|
||||
"attachments": {"target-architecture.json": json.dumps(graph)},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def addition() -> Call:
|
||||
"""A validation that passes every Check and proposes one more."""
|
||||
call = sample_submission(K.VALIDATION_RESULTS)
|
||||
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
verdicts["proposed_additions"] = [
|
||||
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": "NFR-1"}
|
||||
]
|
||||
return Call(
|
||||
call.tool,
|
||||
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
|
||||
)
|
||||
|
||||
|
||||
async def run(tmp_path, second_graph: Call, **options):
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]] * 2
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
turns["target-architect"] = [
|
||||
[sample_submission(K.TARGET_ARCHITECTURE)],
|
||||
[second_graph],
|
||||
]
|
||||
turns["validator"] = [[addition()], [sample_submission(K.VALIDATION_RESULTS)]]
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
**options,
|
||||
)
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
assert result.status == "completed"
|
||||
return result, [run.role for run in runtime.runs]
|
||||
|
||||
|
||||
async def test_the_description_is_kept_when_a_revision_only_changes_checks(tmp_path):
|
||||
reworded = graph_with(
|
||||
checks=lambda checks: [
|
||||
{**checks[0], "statement": "min_instance_count is at least 2"},
|
||||
checks[1],
|
||||
]
|
||||
)
|
||||
|
||||
result, roles_run = await run(tmp_path, reworded)
|
||||
|
||||
assert roles_run.count("target-architect") == 2
|
||||
assert roles_run.count("iac-author") == 2 # the code answers the Checks
|
||||
assert roles_run.count("describer") == 1
|
||||
# The Ledger still records a second submission: the Deliverable was not skipped.
|
||||
submitted = [
|
||||
event
|
||||
for event in result.state["ledger"]["events"]
|
||||
if event["type"] == "submitted"
|
||||
and event["subject"] == "architecture_description"
|
||||
]
|
||||
assert len(submitted) == 2
|
||||
|
||||
|
||||
async def test_the_description_is_written_again_when_the_shape_changes(tmp_path):
|
||||
renamed = graph_with(
|
||||
elements=lambda elements: [{**elements[0], "name": "Edge service"}, elements[1]]
|
||||
)
|
||||
|
||||
_, roles_run = await run(tmp_path, renamed)
|
||||
|
||||
assert roles_run.count("describer") == 2
|
||||
|
||||
|
||||
async def test_the_engine_reviews_a_design_before_it_is_submitted(tmp_path):
|
||||
raised = []
|
||||
|
||||
async def review(kind, attachments, upstream, scope=""):
|
||||
if kind == K.TARGET_ARCHITECTURE and not raised:
|
||||
raised.append(scope)
|
||||
return ['NFR-1 "Two instances." (its Checks: K-1)']
|
||||
return ()
|
||||
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
turns["target-architect"] = [
|
||||
[sample_submission(K.TARGET_ARCHITECTURE)] * 2 # refused once, then accepted
|
||||
]
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
submission_review=review,
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True, execution_id="e1")
|
||||
|
||||
assert result.status == "completed"
|
||||
assert raised == ["e1"]
|
||||
architect = next(run for run in runtime.runs if run.role == "target-architect")
|
||||
assert architect.results[0][1].startswith("Not submitted. A review found")
|
||||
# What a review raises is kept as a lesson candidate, like any refusal.
|
||||
assert any(
|
||||
"A review found" in json.dumps(outcome.get("refusals"))
|
||||
for outcome in result.state["outcomes"]
|
||||
)
|
||||
|
||||
|
||||
async def test_feedback_on_the_description_itself_has_it_written_again(tmp_path):
|
||||
"""Kept only while nobody has asked for the description to change."""
|
||||
from execution_engine.nodes.execution_nodes import STANDING, make_execution_nodes
|
||||
|
||||
nodes = make_execution_nodes(
|
||||
workflow=WORKFLOW,
|
||||
runtime=FakeAgentRuntime(
|
||||
{"describer": [[sample_submission(K.ARCHITECTURE_DESCRIPTION, "new")]]}
|
||||
),
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
output_root=tmp_path,
|
||||
)
|
||||
upstream = {"target_architecture": {"attachments": dict(GRAPH)}}
|
||||
from tests.reference_workflow.chain import graph_structure
|
||||
|
||||
standing = {
|
||||
"basis": graph_structure(upstream),
|
||||
"feedback": [],
|
||||
"content": "old",
|
||||
"attachments": {},
|
||||
}
|
||||
task = {
|
||||
"execution_id": "e1",
|
||||
"role": "describer",
|
||||
"kind": "architecture_description",
|
||||
"attempt": 2,
|
||||
"allow_cloud_read": False,
|
||||
"working": {STANDING: standing},
|
||||
"context": {
|
||||
"dependencies": upstream,
|
||||
"feedback": ["Dependency x was revised: y"],
|
||||
},
|
||||
}
|
||||
|
||||
kept = await nodes.specialist_task(task)
|
||||
asked = await nodes.specialist_task(
|
||||
{**task, "context": {**task["context"], "feedback": ["Make the flow clearer."]}}
|
||||
)
|
||||
|
||||
assert kept["outcomes"][0]["content"] == "old"
|
||||
assert "new" in asked["outcomes"][0]["content"]
|
||||
assert asked["working"]["architecture_description"][STANDING]["feedback"] == [
|
||||
"Make the flow clearer."
|
||||
]
|
||||
|
||||
|
||||
# --- requirements name no products ------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_requirement_that_names_a_product_is_raised():
|
||||
from tests.reference_workflow.requirements_review import RequirementsReview
|
||||
|
||||
requirements = [
|
||||
{"id": "FR-1", "kind": "functional", "text": "Run the API on Cloud Run."},
|
||||
{"id": "FR-2", "kind": "functional", "text": "Accept signed webhooks."},
|
||||
]
|
||||
decider = FakeDecider({"FR-1": Answer(0.93, 0.86), "FR-2": Answer(0.05, 0.9)})
|
||||
|
||||
raised = await RequirementsReview(decider)(
|
||||
K.REQUIREMENTS, {"requirements.json": json.dumps(requirements)}, None, "e1"
|
||||
)
|
||||
|
||||
assert len(raised) == 1 and raised[0].startswith("FR-1 names a product")
|
||||
_, questions, scope = decider.asked[0]
|
||||
assert set(questions) == {"FR-1", "FR-2"} and scope == "e1"
|
||||
assert "Accept signed webhooks." in questions["FR-2"].instructions
|
||||
|
||||
|
||||
async def test_the_requirements_review_looks_only_at_requirements_and_never_blocks():
|
||||
from tests.reference_workflow.requirements_review import RequirementsReview
|
||||
|
||||
decider = FakeDecider({})
|
||||
review = RequirementsReview(decider)
|
||||
|
||||
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
|
||||
assert await review(K.REQUIREMENTS, {"requirements.json": "not json"}, None) == ()
|
||||
assert decider.asked == []
|
||||
assert (
|
||||
await RequirementsReview(NoDecider())(
|
||||
K.REQUIREMENTS,
|
||||
{"requirements.json": SAMPLE_DATA["requirements.json"]},
|
||||
None,
|
||||
)
|
||||
== ()
|
||||
)
|
||||
310
tests/execution/test_deliverables.py
Normal file
310
tests/execution/test_deliverables.py
Normal file
@@ -0,0 +1,310 @@
|
||||
"""The dependency graph and Deliverable contracts."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
PHASE_ORDER,
|
||||
DeliverableKind,
|
||||
PhaseId,
|
||||
check_submission,
|
||||
deliverables_in,
|
||||
inputs_of,
|
||||
)
|
||||
|
||||
|
||||
def test_every_kind_is_declared_once_with_known_dependencies():
|
||||
assert set(DEPENDENCY_GRAPH) == set(DeliverableKind)
|
||||
for kind, spec in DEPENDENCY_GRAPH.items():
|
||||
assert spec.kind is kind
|
||||
assert set(spec.depends_on) <= set(DeliverableKind), kind
|
||||
|
||||
|
||||
def test_dependencies_never_point_to_a_later_phase_and_form_no_cycle():
|
||||
position = {phase: index for index, phase in enumerate(PHASE_ORDER)}
|
||||
for spec in DEPENDENCY_GRAPH.values():
|
||||
for dependency in spec.depends_on:
|
||||
assert position[DEPENDENCY_GRAPH[dependency].phase] <= position[spec.phase]
|
||||
|
||||
visiting: set[str] = set()
|
||||
done: set[str] = set()
|
||||
|
||||
def visit(kind: str) -> None:
|
||||
assert kind not in visiting, f"cycle through {kind}"
|
||||
if kind in done:
|
||||
return
|
||||
visiting.add(kind)
|
||||
for dependency in DEPENDENCY_GRAPH[kind].depends_on:
|
||||
visit(dependency)
|
||||
visiting.discard(kind)
|
||||
done.add(kind)
|
||||
|
||||
for kind in DeliverableKind:
|
||||
visit(kind)
|
||||
|
||||
|
||||
def test_every_phase_groups_at_least_one_deliverable_with_a_distinct_specialist():
|
||||
assert PHASE_ORDER == (
|
||||
PhaseId.SCOPING,
|
||||
PhaseId.REQUIREMENTS_DISCOVERY,
|
||||
PhaseId.SOLUTION_ARCHITECTURE,
|
||||
PhaseId.SOLUTION_IMPLEMENTATION,
|
||||
PhaseId.SOLUTION_VALIDATION,
|
||||
PhaseId.PACKAGING,
|
||||
)
|
||||
for phase in PHASE_ORDER:
|
||||
assert deliverables_in(phase), phase
|
||||
specialists = [spec.specialist for spec in DEPENDENCY_GRAPH.values()]
|
||||
assert len(specialists) == len(set(specialists))
|
||||
|
||||
|
||||
def test_the_target_architecture_sits_between_the_design_and_the_code():
|
||||
depends = {kind: set(spec.depends_on) for kind, spec in DEPENDENCY_GRAPH.items()}
|
||||
K = DeliverableKind
|
||||
|
||||
assert depends[K.SCOPE] == set()
|
||||
assert depends[K.REQUIREMENTS] == {K.SCOPE}
|
||||
assert depends[K.TECHNICAL_DECOMPOSITION] == {K.REQUIREMENTS}
|
||||
assert depends[K.PRODUCT_SELECTION] == {K.TECHNICAL_DECOMPOSITION}
|
||||
assert depends[K.DESIGN_RECOMMENDATIONS] == {K.PRODUCT_SELECTION, K.REQUIREMENTS}
|
||||
assert depends[K.TARGET_ARCHITECTURE] == {
|
||||
K.PRODUCT_SELECTION,
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
}
|
||||
# Everything after the graph is derived from it.
|
||||
assert depends[K.ARCHITECTURE_DESCRIPTION] == {K.TARGET_ARCHITECTURE}
|
||||
assert depends[K.DEPLOYMENT_GUIDANCE] == {K.TARGET_ARCHITECTURE}
|
||||
assert depends[K.VALIDATION_RESULTS] == {K.DEPLOYMENT_GUIDANCE}
|
||||
assert depends[K.SOLUTION_GUIDE] == set(
|
||||
deliverables_in(PhaseId.SOLUTION_VALIDATION)
|
||||
)
|
||||
|
||||
|
||||
def test_the_design_is_approved_before_code_is_written():
|
||||
design = PhaseId.SOLUTION_ARCHITECTURE
|
||||
implementation = PhaseId.SOLUTION_IMPLEMENTATION
|
||||
|
||||
assert DEPENDENCY_GRAPH[DeliverableKind.TARGET_ARCHITECTURE].phase is design
|
||||
assert DEPENDENCY_GRAPH[DeliverableKind.DEPLOYMENT_GUIDANCE].phase is implementation
|
||||
assert PHASE_ORDER.index(design) < PHASE_ORDER.index(implementation)
|
||||
|
||||
|
||||
def test_inputs_are_earlier_phases_plus_indirect_dependencies_in_graph_order():
|
||||
design = (
|
||||
DeliverableKind.SCOPE,
|
||||
DeliverableKind.REQUIREMENTS,
|
||||
DeliverableKind.TECHNICAL_DECOMPOSITION,
|
||||
DeliverableKind.PRODUCT_SELECTION,
|
||||
DeliverableKind.DESIGN_RECOMMENDATIONS,
|
||||
DeliverableKind.TARGET_ARCHITECTURE,
|
||||
)
|
||||
assert inputs_of(DeliverableKind.SCOPE) == ()
|
||||
assert inputs_of(DeliverableKind.REQUIREMENTS) == (DeliverableKind.SCOPE,)
|
||||
assert inputs_of(DeliverableKind.DEPLOYMENT_GUIDANCE) == design
|
||||
assert inputs_of(DeliverableKind.ARCHITECTURE_DESCRIPTION) == design
|
||||
assert inputs_of(DeliverableKind.TARGET_ARCHITECTURE) == design[:5]
|
||||
# The plan is produced beside the guide, not before it: neither reads the other.
|
||||
assert set(inputs_of(DeliverableKind.SOLUTION_GUIDE)) == set(DeliverableKind) - {
|
||||
DeliverableKind.SOLUTION_GUIDE,
|
||||
DeliverableKind.WORK_BREAKDOWN,
|
||||
}
|
||||
assert DeliverableKind.SOLUTION_GUIDE not in inputs_of(
|
||||
DeliverableKind.WORK_BREAKDOWN
|
||||
)
|
||||
|
||||
|
||||
def sections_missing(kind: DeliverableKind, content: str) -> list[str]:
|
||||
return [m for m in check_submission(kind, content, {}) if m.startswith("section:")]
|
||||
|
||||
|
||||
def test_submission_meeting_the_contract_has_nothing_missing():
|
||||
content = (
|
||||
"# Deployment guidance\n"
|
||||
"## 6.1. Deployment prerequisites\nEnable APIs.\n"
|
||||
"## 6.2. Step-by-step deployment instructions\nRun terraform.\n"
|
||||
)
|
||||
attachments = {
|
||||
"terraform/main.tf": "resource {}",
|
||||
"terraform/variables.tf": "variable {}",
|
||||
"implementation.json": "[]",
|
||||
}
|
||||
|
||||
assert (
|
||||
check_submission(DeliverableKind.DEPLOYMENT_GUIDANCE, content, attachments)
|
||||
== ()
|
||||
)
|
||||
|
||||
|
||||
def test_submission_reports_missing_sections_and_attachments():
|
||||
missing = check_submission(
|
||||
DeliverableKind.DEPLOYMENT_GUIDANCE,
|
||||
"## Deployment prerequisites\n",
|
||||
{"terraform/main.tf": "resource {}"},
|
||||
)
|
||||
|
||||
assert missing == (
|
||||
"section: Step-by-step deployment instructions",
|
||||
"attachment: terraform/variables.tf",
|
||||
"attachment: implementation.json",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"heading",
|
||||
[
|
||||
"### 5.1. Security, privacy, and compliance",
|
||||
"## security, privacy, and compliance",
|
||||
"# 5.1 Security, Privacy, and Compliance:",
|
||||
],
|
||||
)
|
||||
def test_section_matching_ignores_numbering_case_and_trailing_punctuation(heading):
|
||||
content = (
|
||||
f"{heading}\n"
|
||||
"## Reliability\n"
|
||||
"## Operational excellence\n"
|
||||
"## Cost optimization\n"
|
||||
"## Performance efficiency\n"
|
||||
"## Sustainability\n"
|
||||
)
|
||||
|
||||
assert sections_missing(DeliverableKind.DESIGN_RECOMMENDATIONS, content) == []
|
||||
|
||||
|
||||
def test_section_matches_a_provider_prefixed_heading_but_not_a_longer_word():
|
||||
content = (
|
||||
"### 4.1. Google Cloud products and features mapping\n"
|
||||
"## Non-functional requirements\n"
|
||||
)
|
||||
|
||||
assert sections_missing(DeliverableKind.PRODUCT_SELECTION, content) == []
|
||||
assert "section: Functional requirements" in check_submission(
|
||||
DeliverableKind.REQUIREMENTS, content, {}
|
||||
)
|
||||
|
||||
|
||||
def test_section_names_in_body_text_do_not_count_as_headings():
|
||||
missing = check_submission(
|
||||
DeliverableKind.REQUIREMENTS,
|
||||
"Functional requirements, non-functional requirements, current state and "
|
||||
"dependencies are all covered below.\n",
|
||||
{},
|
||||
)
|
||||
|
||||
assert "section: Functional requirements" in missing
|
||||
|
||||
|
||||
def test_empty_content_is_refused():
|
||||
assert check_submission(DeliverableKind.TECHNICAL_DECOMPOSITION, " \n", {}) == (
|
||||
"content: must not be empty",
|
||||
)
|
||||
|
||||
|
||||
# --- contracts beyond headings ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_architecture_description_needs_component_and_flow_sections():
|
||||
missing = check_submission(
|
||||
DeliverableKind.ARCHITECTURE_DESCRIPTION, "The system works.", {}
|
||||
)
|
||||
|
||||
assert missing == ("section: Components", "section: Flow")
|
||||
assert (
|
||||
check_submission(
|
||||
DeliverableKind.ARCHITECTURE_DESCRIPTION,
|
||||
"## Architecture components\nA.\n## Data flow\nB.\n",
|
||||
{},
|
||||
)
|
||||
== ()
|
||||
)
|
||||
|
||||
|
||||
def test_a_required_attachment_must_have_content():
|
||||
missing = check_submission(
|
||||
DeliverableKind.VALIDATION_RESULTS,
|
||||
"## Deployment dry-run\nPass.\n## Architecture and policy analysis\nPass.\n",
|
||||
{"verdicts.json": " \n"},
|
||||
)
|
||||
|
||||
assert missing == ("attachment: verdicts.json must not be empty",)
|
||||
|
||||
|
||||
# --- what a Specialist is shown of what it is given --------------------------------------
|
||||
|
||||
|
||||
def given_for(kind: DeliverableKind) -> dict:
|
||||
from tests.reference_workflow.workflow import dependencies_for, sample_submission
|
||||
|
||||
submitted = {}
|
||||
for each in DeliverableKind:
|
||||
call = sample_submission(each)
|
||||
submitted[each.value] = {
|
||||
"content": call.args["content"],
|
||||
"attachments": call.args["attachments"],
|
||||
}
|
||||
review = "\n## Architecture framework checklist review\n| a | b |\n"
|
||||
submitted["design_recommendations"]["content"] += review
|
||||
return dependencies_for(kind, submitted)
|
||||
|
||||
|
||||
def test_a_specialist_is_shown_only_what_it_uses():
|
||||
from tests.reference_workflow.workflow import shown
|
||||
|
||||
author = shown(
|
||||
DeliverableKind.DEPLOYMENT_GUIDANCE,
|
||||
given_for(DeliverableKind.DEPLOYMENT_GUIDANCE),
|
||||
)
|
||||
validator = shown(
|
||||
DeliverableKind.VALIDATION_RESULTS,
|
||||
given_for(DeliverableKind.VALIDATION_RESULTS),
|
||||
)
|
||||
|
||||
# The graph is all the code is written from.
|
||||
assert list(author) == ["target_architecture"]
|
||||
# The validator judges code against Checks, and reads requirements and
|
||||
# recommendations for coverage; the prose describing the design is not judged.
|
||||
assert "architecture_description" not in validator
|
||||
assert {"requirements", "design_recommendations", "deployment_guidance"} <= set(
|
||||
validator
|
||||
)
|
||||
|
||||
|
||||
def test_nothing_is_taken_from_what_a_submission_is_checked_against():
|
||||
"""Showing less changes what the model reads, never what the code checks."""
|
||||
given = given_for(DeliverableKind.DEPLOYMENT_GUIDANCE)
|
||||
|
||||
assert {"requirements", "design_recommendations", "scope"} <= set(given)
|
||||
|
||||
|
||||
def test_the_scope_is_shown_once_and_only_its_prose_to_who_works_from_it():
|
||||
from tests.reference_workflow.workflow import shown
|
||||
|
||||
analyst = shown(
|
||||
DeliverableKind.REQUIREMENTS, given_for(DeliverableKind.REQUIREMENTS)
|
||||
)
|
||||
architect = shown(
|
||||
DeliverableKind.TARGET_ARCHITECTURE,
|
||||
given_for(DeliverableKind.TARGET_ARCHITECTURE),
|
||||
)
|
||||
|
||||
assert list(analyst["scope"]) == ["content"]
|
||||
assert "scope" not in architect
|
||||
|
||||
|
||||
def test_a_section_written_for_a_reader_is_left_off_except_for_the_guide():
|
||||
from tests.reference_workflow.pillars import REVIEW_HEADING
|
||||
from tests.reference_workflow.workflow import FOR_READERS, shown
|
||||
|
||||
assert FOR_READERS[DeliverableKind.DESIGN_RECOMMENDATIONS] == REVIEW_HEADING
|
||||
architect = shown(
|
||||
DeliverableKind.TARGET_ARCHITECTURE,
|
||||
given_for(DeliverableKind.TARGET_ARCHITECTURE),
|
||||
)
|
||||
packager = shown(
|
||||
DeliverableKind.SOLUTION_GUIDE, given_for(DeliverableKind.SOLUTION_GUIDE)
|
||||
)
|
||||
|
||||
assert REVIEW_HEADING not in architect["design_recommendations"]["content"]
|
||||
assert "## Design" in architect["design_recommendations"]["content"] or architect[
|
||||
"design_recommendations"
|
||||
]["content"].startswith("# design_recommendations")
|
||||
assert REVIEW_HEADING in packager["design_recommendations"]["content"]
|
||||
1082
tests/execution/test_engine.py
Normal file
1082
tests/execution/test_engine.py
Normal file
File diff suppressed because it is too large
Load Diff
78
tests/execution/test_file_arguments.py
Normal file
78
tests/execution/test_file_arguments.py
Normal file
@@ -0,0 +1,78 @@
|
||||
"""Files cross tool calls as a list of {path, content}: models rewrite object keys."""
|
||||
|
||||
import pytest
|
||||
|
||||
from execution_engine.adk.runtime import FILES_SCHEMA, Submitted, files_from_argument
|
||||
from execution_engine.tools.execution import specialist_tools
|
||||
from tests.reference_workflow.local import TerraformRunner, local_tools
|
||||
from tests.reference_workflow.workflow import WORKFLOW, DeliverableKind
|
||||
|
||||
GUIDANCE = (
|
||||
"# Deployment prerequisites\n\nx\n\n# Step-by-step deployment instructions\n\ny\n"
|
||||
)
|
||||
|
||||
|
||||
def test_files_are_offered_as_a_list_of_path_and_content():
|
||||
assert FILES_SCHEMA["type"] == "array"
|
||||
assert FILES_SCHEMA["items"]["required"] == ["path", "content"]
|
||||
|
||||
|
||||
def test_file_arguments_accept_lists_and_maps():
|
||||
as_list = [{"path": "terraform/main.tf", "content": "a"}]
|
||||
|
||||
assert files_from_argument(as_list) == {"terraform/main.tf": "a"}
|
||||
assert files_from_argument({"terraform/main.tf": "a"}) == {"terraform/main.tf": "a"}
|
||||
assert files_from_argument(None) == {}
|
||||
with pytest.raises(TypeError):
|
||||
files_from_argument(["terraform/main.tf"])
|
||||
|
||||
|
||||
async def test_submission_takes_attachments_as_a_list():
|
||||
submit = specialist_tools(
|
||||
WORKFLOW, DeliverableKind.DEPLOYMENT_GUIDANCE, extra=[], allow_cloud_read=False
|
||||
)[0]
|
||||
|
||||
result = await submit.handler(
|
||||
content=GUIDANCE,
|
||||
attachments=[
|
||||
{"path": "terraform/main.tf", "content": "resource {}"},
|
||||
{"path": "terraform/variables.tf", "content": "variable {}"},
|
||||
{"path": "implementation.json", "content": "[]"},
|
||||
],
|
||||
)
|
||||
|
||||
assert result.outcome == Submitted(
|
||||
GUIDANCE,
|
||||
{
|
||||
"terraform/main.tf": "resource {}",
|
||||
"terraform/variables.tf": "variable {}",
|
||||
"implementation.json": "[]",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def test_malformed_attachments_are_refused_to_the_agent():
|
||||
submit = specialist_tools(
|
||||
WORKFLOW, DeliverableKind.DEPLOYMENT_GUIDANCE, extra=[], allow_cloud_read=False
|
||||
)[0]
|
||||
|
||||
result = await submit.handler(content=GUIDANCE, attachments=["terraform/main.tf"])
|
||||
|
||||
assert result.outcome is None
|
||||
assert result.message.startswith("Refused: attachments")
|
||||
|
||||
|
||||
async def test_terraform_tools_take_files_as_a_list():
|
||||
received = {}
|
||||
|
||||
class Runner(TerraformRunner):
|
||||
async def validate(self, files):
|
||||
received.update(files)
|
||||
return await super().validate({})
|
||||
|
||||
(validate,) = local_tools(["terraform_validate"], Runner())
|
||||
|
||||
await validate.handler(files=[{"path": "terraform/main.tf", "content": "x"}])
|
||||
|
||||
assert validate.parameters["properties"]["files"]["type"] == "array"
|
||||
assert received == {"terraform/main.tf": "x"}
|
||||
172
tests/execution/test_intake.py
Normal file
172
tests/execution/test_intake.py
Normal file
@@ -0,0 +1,172 @@
|
||||
"""Intake: what a message that starts something is asking for."""
|
||||
|
||||
import json
|
||||
|
||||
from execution_engine.adk.fake_runtime import FakeDecider
|
||||
from execution_engine.adk.runtime import Answer, NoDecider
|
||||
from execution_engine.nodes.intake import CHANGE_REQUEST, NEW_REQUEST, QUESTION, Intake
|
||||
from tests.reference_workflow.workflow import WORKFLOW
|
||||
|
||||
|
||||
def delivered(tmp_path, case: str, statuses: list[str]) -> None:
|
||||
folder = tmp_path / "deliverables" / "cases" / case / "intent"
|
||||
folder.mkdir(parents=True)
|
||||
intents = [
|
||||
{
|
||||
"version": n,
|
||||
"workflow_request": f"Design thing {n}.",
|
||||
"status": status,
|
||||
"execution_id": f"v{n}",
|
||||
}
|
||||
for n, status in enumerate(statuses, start=1)
|
||||
]
|
||||
(folder / "history.json").write_text(
|
||||
json.dumps({"customer_case": case, "intents": intents})
|
||||
)
|
||||
|
||||
|
||||
async def test_without_a_decision_every_message_is_a_new_request(tmp_path):
|
||||
reading = await Intake(NoDecider(), tmp_path).read("What can you do?")
|
||||
|
||||
assert reading.kind == NEW_REQUEST and not reading.judged
|
||||
|
||||
|
||||
async def test_a_confident_question_is_read_as_one(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer(QUESTION, 0.98)})
|
||||
|
||||
reading = await Intake(decider, tmp_path).read("How long does it take?", scope="t1")
|
||||
|
||||
state, questions, scope = decider.asked[0]
|
||||
assert reading.kind == QUESTION and reading.judged
|
||||
assert scope == "t1"
|
||||
assert questions["intake"].options == (NEW_REQUEST, CHANGE_REQUEST, QUESTION)
|
||||
assert state.endswith("The user's message: How long does it take?")
|
||||
|
||||
|
||||
async def test_a_reading_that_is_not_sure_enough_starts_new_work(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer(QUESTION, 0.6)})
|
||||
|
||||
assert (await Intake(decider, tmp_path, act=0.75).read("hello")).kind == NEW_REQUEST
|
||||
|
||||
|
||||
async def test_a_change_needs_something_delivered_to_change(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer(CHANGE_REQUEST, 0.99)})
|
||||
delivered(tmp_path, "acme", ["completed", "in_progress"])
|
||||
intake = Intake(decider, tmp_path)
|
||||
|
||||
with_history = await intake.read("Raise availability to 99.99%.", "acme")
|
||||
without = await intake.read("Raise availability to 99.99%.", "new-client")
|
||||
no_case = await intake.read("Raise availability to 99.99%.")
|
||||
|
||||
assert with_history.kind == CHANGE_REQUEST and with_history.earlier == 1
|
||||
assert with_history.latest == "v1" # what the change would change
|
||||
assert "- Design thing 1." in decider.asked[0][0]
|
||||
assert "Design thing 2." not in decider.asked[0][0] # still in progress
|
||||
assert without.kind == NEW_REQUEST and no_case.kind == NEW_REQUEST
|
||||
|
||||
|
||||
# --- a question about what was delivered ------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_question_about_delivered_work_is_told_from_one_about_the_service(
|
||||
tmp_path,
|
||||
):
|
||||
delivered(tmp_path, "acme", ["completed"])
|
||||
about = FakeDecider(
|
||||
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.9, 0.9)}
|
||||
)
|
||||
general = FakeDecider(
|
||||
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.1, 0.9)}
|
||||
)
|
||||
|
||||
first = await Intake(about, tmp_path).read("Why Cloud SQL?", "acme")
|
||||
second = await Intake(general, tmp_path).read("What can you do?", "acme")
|
||||
|
||||
assert first.kind == QUESTION and first.about_delivered and first.latest == "v1"
|
||||
assert second.kind == QUESTION and not second.about_delivered
|
||||
|
||||
|
||||
async def test_with_nothing_delivered_a_question_is_never_about_delivered_work(
|
||||
tmp_path,
|
||||
):
|
||||
decider = FakeDecider(
|
||||
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.99, 0.9)}
|
||||
)
|
||||
|
||||
reading = await Intake(decider, tmp_path).read("Why Cloud SQL?", "new-client")
|
||||
|
||||
assert not reading.about_delivered
|
||||
assert len(decider.asked) == 1 # the second question was not asked
|
||||
|
||||
|
||||
async def test_an_answer_is_written_from_the_delivered_guide_and_nothing_else(
|
||||
tmp_path,
|
||||
):
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import (
|
||||
Call,
|
||||
FakeAgentRuntime,
|
||||
)
|
||||
from execution_engine.adk.runtime import RoleSpec
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.nodes.intake import DeliveredAnswers
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
sample_submission,
|
||||
)
|
||||
|
||||
roles = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
seen: dict = {}
|
||||
|
||||
def writer(task):
|
||||
seen.update(task.context)
|
||||
return [Call("write_reply", {"text": "Cloud SQL, for relational bookings."})]
|
||||
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
**{
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
},
|
||||
"reply-writer": [writer],
|
||||
}
|
||||
)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
await engine.start(
|
||||
"Design bookings.", customer_case="acme", pre_approved=True, execution_id="v1"
|
||||
)
|
||||
runs_before = len(runtime.runs)
|
||||
answers = DeliveredAnswers(
|
||||
runtime,
|
||||
RoleSpec("reply-writer", "Write.", budget=3),
|
||||
tmp_path,
|
||||
"solution_guide",
|
||||
)
|
||||
|
||||
answer = await answers.answer("Why Cloud SQL?", "acme", "v1")
|
||||
|
||||
assert answer == "Cloud SQL, for relational bookings."
|
||||
assert seen["question"] == "Why Cloud SQL?"
|
||||
assert seen["delivered"]["execution_id"] == "v1"
|
||||
assert seen["delivered"]["request"] == "Design bookings."
|
||||
assert seen["delivered"]["final_document"]
|
||||
assert len(runtime.runs) == runs_before + 1 # one writer run; no Execution
|
||||
# An Execution that is not there gives no answer, not an error.
|
||||
assert await answers.answer("Why?", "acme", "missing") is None
|
||||
397
tests/execution/test_ledger.py
Normal file
397
tests/execution/test_ledger.py
Normal file
@@ -0,0 +1,397 @@
|
||||
"""The Ledger: Dependency acceptance, Approvals and Revision rules."""
|
||||
|
||||
import pytest
|
||||
|
||||
from execution_engine.states.ledger import (
|
||||
DeliverableStatus,
|
||||
Ledger,
|
||||
LedgerRefusal,
|
||||
PhaseStatus,
|
||||
)
|
||||
from tests.reference_workflow.workflow import WORKFLOW
|
||||
from tests.reference_workflow.workflow import DeliverableKind as K
|
||||
from tests.reference_workflow.workflow import PhaseId as P
|
||||
|
||||
|
||||
def produce(ledger: Ledger, *kinds: K) -> None:
|
||||
"""Delegate and submit each kind, in order."""
|
||||
for kind in kinds:
|
||||
attempt = ledger.start([kind], actor="orchestrator")[kind]
|
||||
ledger.submit(kind, attempt, actor=f"specialist:{kind}")
|
||||
|
||||
|
||||
def approve(ledger: Ledger, phase: P) -> None:
|
||||
ledger.request_approval(phase, actor="orchestrator")
|
||||
if ledger.phase_status(phase) is PhaseStatus.AWAITING_APPROVAL:
|
||||
ledger.approve(phase, actor="user")
|
||||
|
||||
|
||||
def scoped(**options) -> Ledger:
|
||||
"""A Ledger whose scope is agreed: where requirements discovery begins."""
|
||||
ledger = Ledger(WORKFLOW, **options)
|
||||
produce(ledger, K.SCOPE)
|
||||
approve(ledger, P.SCOPING)
|
||||
return ledger
|
||||
|
||||
|
||||
def through_phase_one(ledger: Ledger) -> None:
|
||||
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
|
||||
approve(ledger, P.REQUIREMENTS_DISCOVERY)
|
||||
|
||||
|
||||
def through_phase_two(ledger: Ledger) -> None:
|
||||
through_phase_one(ledger)
|
||||
produce(
|
||||
ledger, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE
|
||||
)
|
||||
approve(ledger, P.SOLUTION_ARCHITECTURE)
|
||||
produce(ledger, K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)
|
||||
approve(ledger, P.SOLUTION_IMPLEMENTATION)
|
||||
|
||||
|
||||
def through_phase_three(ledger: Ledger) -> None:
|
||||
through_phase_two(ledger)
|
||||
produce(ledger, K.VALIDATION_RESULTS)
|
||||
approve(ledger, P.SOLUTION_VALIDATION)
|
||||
|
||||
|
||||
# --- ready work and the two acceptance thresholds -------------------------------
|
||||
|
||||
|
||||
def test_a_new_execution_has_only_the_scope_ready():
|
||||
ledger = Ledger(WORKFLOW)
|
||||
|
||||
assert ledger.ready_work() == (K.SCOPE,)
|
||||
assert all(ledger.phase_status(p) is PhaseStatus.OPEN for p in P)
|
||||
|
||||
|
||||
def test_once_the_scope_is_approved_requirements_are_ready():
|
||||
ledger = scoped()
|
||||
|
||||
assert ledger.ready_work() == (K.REQUIREMENTS,)
|
||||
assert ledger.phase_status(P.SCOPING) is PhaseStatus.APPROVED
|
||||
|
||||
|
||||
def test_same_phase_dependency_is_met_once_submitted():
|
||||
ledger = scoped()
|
||||
produce(ledger, K.REQUIREMENTS)
|
||||
|
||||
assert ledger.ready_work() == (K.TECHNICAL_DECOMPOSITION,)
|
||||
|
||||
|
||||
def test_cross_phase_dependency_needs_the_earlier_phase_approved():
|
||||
ledger = scoped()
|
||||
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
|
||||
|
||||
assert K.PRODUCT_SELECTION not in ledger.ready_work()
|
||||
|
||||
approve(ledger, P.REQUIREMENTS_DISCOVERY)
|
||||
|
||||
assert ledger.ready_work() == (K.PRODUCT_SELECTION,)
|
||||
|
||||
|
||||
def test_independent_work_becomes_ready_together():
|
||||
ledger = scoped()
|
||||
through_phase_one(ledger)
|
||||
produce(
|
||||
ledger, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, K.TARGET_ARCHITECTURE
|
||||
)
|
||||
|
||||
# Nothing in implementation is ready until the design is approved.
|
||||
assert ledger.ready_work() == ()
|
||||
|
||||
approve(ledger, P.SOLUTION_ARCHITECTURE)
|
||||
|
||||
assert set(ledger.ready_work()) == {
|
||||
K.ARCHITECTURE_DESCRIPTION,
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
}
|
||||
|
||||
|
||||
def test_ready_phases_lists_open_phases_with_every_deliverable_submitted():
|
||||
ledger = scoped()
|
||||
produce(ledger, K.REQUIREMENTS)
|
||||
assert ledger.phases_ready_for_approval() == ()
|
||||
|
||||
produce(ledger, K.TECHNICAL_DECOMPOSITION)
|
||||
|
||||
assert ledger.phases_ready_for_approval() == (P.REQUIREMENTS_DISCOVERY,)
|
||||
|
||||
|
||||
# --- delegation and submission ---------------------------------------------------
|
||||
|
||||
|
||||
def test_start_refuses_work_that_is_not_ready_and_changes_nothing():
|
||||
ledger = scoped()
|
||||
|
||||
with pytest.raises(LedgerRefusal, match="technical_decomposition"):
|
||||
ledger.start([K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION], actor="orchestrator")
|
||||
|
||||
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.PENDING
|
||||
|
||||
|
||||
def test_start_moves_ready_work_in_progress_and_returns_attempts():
|
||||
ledger = scoped()
|
||||
|
||||
attempts = ledger.start([K.REQUIREMENTS], actor="orchestrator")
|
||||
|
||||
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.IN_PROGRESS
|
||||
assert attempts == {K.REQUIREMENTS: 1}
|
||||
assert ledger.ready_work() == ()
|
||||
|
||||
|
||||
def test_submit_requires_the_current_attempt():
|
||||
ledger = scoped()
|
||||
attempt = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
|
||||
|
||||
with pytest.raises(LedgerRefusal, match="attempt"):
|
||||
ledger.submit(K.REQUIREMENTS, attempt + 1, actor="requirements-analyst")
|
||||
|
||||
ledger.submit(K.REQUIREMENTS, attempt, actor="requirements-analyst")
|
||||
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED
|
||||
|
||||
|
||||
def test_late_submission_from_a_superseded_attempt_is_refused_after_restart():
|
||||
ledger = scoped()
|
||||
first = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
|
||||
ledger.release(K.REQUIREMENTS, first, reason="needs input")
|
||||
second = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
|
||||
|
||||
with pytest.raises(LedgerRefusal, match="attempt"):
|
||||
ledger.submit(K.REQUIREMENTS, first, actor="requirements-analyst")
|
||||
|
||||
ledger.submit(K.REQUIREMENTS, second, actor="requirements-analyst")
|
||||
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED
|
||||
|
||||
|
||||
def test_submit_refuses_work_not_in_progress():
|
||||
ledger = scoped()
|
||||
|
||||
with pytest.raises(LedgerRefusal, match="in_progress"):
|
||||
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
|
||||
|
||||
|
||||
def test_release_returns_work_to_pending_without_a_revision():
|
||||
ledger = scoped()
|
||||
attempt = ledger.start([K.REQUIREMENTS], actor="orchestrator")[K.REQUIREMENTS]
|
||||
|
||||
ledger.release(K.REQUIREMENTS, attempt, reason="needs input")
|
||||
|
||||
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.PENDING
|
||||
assert ledger.revisions(K.REQUIREMENTS) == 0
|
||||
assert ledger.ready_work() == (K.REQUIREMENTS,)
|
||||
with pytest.raises(LedgerRefusal, match="attempt"):
|
||||
ledger.submit(K.REQUIREMENTS, attempt, actor="requirements-analyst")
|
||||
|
||||
|
||||
# --- approvals ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_request_approval_refuses_until_every_deliverable_is_submitted():
|
||||
ledger = scoped()
|
||||
produce(ledger, K.REQUIREMENTS)
|
||||
|
||||
with pytest.raises(LedgerRefusal, match="technical_decomposition"):
|
||||
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
|
||||
|
||||
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.OPEN
|
||||
|
||||
|
||||
def test_approval_moves_through_awaiting_to_approved_by_the_user():
|
||||
ledger = scoped()
|
||||
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
|
||||
|
||||
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
|
||||
assert (
|
||||
ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.AWAITING_APPROVAL
|
||||
)
|
||||
|
||||
ledger.approve(P.REQUIREMENTS_DISCOVERY, actor="user")
|
||||
|
||||
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
|
||||
assert ledger.events[-1].type == "approved"
|
||||
assert ledger.events[-1].actor == "user"
|
||||
|
||||
|
||||
def test_approve_refuses_a_phase_not_awaiting_approval():
|
||||
ledger = scoped()
|
||||
|
||||
with pytest.raises(LedgerRefusal, match="awaiting_approval"):
|
||||
ledger.approve(P.REQUIREMENTS_DISCOVERY, actor="user")
|
||||
|
||||
|
||||
def test_pre_approved_ledger_grants_approval_immediately():
|
||||
ledger = scoped(pre_approved=True)
|
||||
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
|
||||
|
||||
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
|
||||
|
||||
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
|
||||
assert [e.type for e in ledger.events[-2:]] == ["approval_requested", "approved"]
|
||||
assert ledger.events[-1].actor == "pre-approved"
|
||||
|
||||
|
||||
def test_request_changes_reopens_the_phase_for_further_work():
|
||||
ledger = scoped()
|
||||
produce(ledger, K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
|
||||
ledger.request_approval(P.REQUIREMENTS_DISCOVERY, actor="orchestrator")
|
||||
|
||||
ledger.request_changes(P.REQUIREMENTS_DISCOVERY, "Add the RTO target", actor="user")
|
||||
|
||||
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.OPEN
|
||||
assert ledger.events[-1].type == "changes_requested"
|
||||
assert ledger.events[-1].detail == "Add the RTO target"
|
||||
|
||||
|
||||
def test_finish_is_allowed_only_once_packaging_is_approved():
|
||||
ledger = scoped()
|
||||
through_phase_three(ledger)
|
||||
assert not ledger.can_finish()
|
||||
|
||||
produce(ledger, K.WORK_BREAKDOWN, K.SOLUTION_GUIDE)
|
||||
approve(ledger, P.PACKAGING)
|
||||
|
||||
assert ledger.can_finish()
|
||||
|
||||
|
||||
# --- revisions ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_reopen_cascades_to_dependents_and_withdraws_affected_approvals():
|
||||
ledger = scoped()
|
||||
through_phase_three(ledger)
|
||||
|
||||
result = ledger.reopen(
|
||||
K.DEPLOYMENT_GUIDANCE, "terraform plan failed", actor="orchestrator"
|
||||
)
|
||||
|
||||
assert result.escalation is None
|
||||
# solution_guide was still pending, so its status doesn't change and it isn't reset.
|
||||
assert set(result.reset) == {K.DEPLOYMENT_GUIDANCE, K.VALIDATION_RESULTS}
|
||||
assert ledger.status(K.DEPLOYMENT_GUIDANCE) is DeliverableStatus.PENDING
|
||||
assert ledger.status(K.VALIDATION_RESULTS) is DeliverableStatus.PENDING
|
||||
assert ledger.status(K.PRODUCT_SELECTION) is DeliverableStatus.SUBMITTED
|
||||
assert ledger.phase_status(P.REQUIREMENTS_DISCOVERY) is PhaseStatus.APPROVED
|
||||
# Revising the code leaves the approved design standing.
|
||||
assert ledger.phase_status(P.SOLUTION_ARCHITECTURE) is PhaseStatus.APPROVED
|
||||
assert ledger.phase_status(P.SOLUTION_IMPLEMENTATION) is PhaseStatus.OPEN
|
||||
assert ledger.phase_status(P.SOLUTION_VALIDATION) is PhaseStatus.OPEN
|
||||
assert set(result.withdrawn_approvals) == {
|
||||
P.SOLUTION_IMPLEMENTATION,
|
||||
P.SOLUTION_VALIDATION,
|
||||
}
|
||||
assert ledger.revisions(K.DEPLOYMENT_GUIDANCE) == 1
|
||||
assert ledger.revisions(K.VALIDATION_RESULTS) == 0
|
||||
assert ledger.feedback(K.DEPLOYMENT_GUIDANCE) == ("terraform plan failed",)
|
||||
assert ledger.feedback(K.VALIDATION_RESULTS) == (
|
||||
"Dependency deployment_guidance was revised: terraform plan failed",
|
||||
)
|
||||
assert ledger.ready_work() == (K.DEPLOYMENT_GUIDANCE,)
|
||||
|
||||
|
||||
def test_reopen_invalidates_attempts_of_dependents_already_in_progress():
|
||||
ledger = scoped()
|
||||
through_phase_one(ledger)
|
||||
produce(ledger, K.PRODUCT_SELECTION)
|
||||
attempt = ledger.start([K.DESIGN_RECOMMENDATIONS], actor="orchestrator")[
|
||||
K.DESIGN_RECOMMENDATIONS
|
||||
]
|
||||
|
||||
ledger.reopen(K.PRODUCT_SELECTION, "user prefers GKE", actor="orchestrator")
|
||||
|
||||
assert ledger.status(K.DESIGN_RECOMMENDATIONS) is DeliverableStatus.PENDING
|
||||
with pytest.raises(LedgerRefusal, match="attempt"):
|
||||
ledger.submit(K.DESIGN_RECOMMENDATIONS, attempt, actor="design-advisor")
|
||||
|
||||
|
||||
def test_reopen_refuses_deliverables_that_were_never_submitted():
|
||||
ledger = scoped()
|
||||
|
||||
with pytest.raises(LedgerRefusal, match="submitted"):
|
||||
ledger.reopen(K.REQUIREMENTS, "why", actor="orchestrator")
|
||||
|
||||
|
||||
def test_reopen_escalates_at_the_revision_cap_without_changing_state():
|
||||
ledger = scoped(max_revisions=2)
|
||||
produce(ledger, K.REQUIREMENTS)
|
||||
for _ in range(2):
|
||||
ledger.reopen(K.REQUIREMENTS, "again", actor="orchestrator")
|
||||
produce(ledger, K.REQUIREMENTS)
|
||||
|
||||
result = ledger.reopen(K.REQUIREMENTS, "one more", actor="orchestrator")
|
||||
|
||||
assert result.reset == ()
|
||||
assert result.escalation is not None
|
||||
assert "requirements" in result.escalation
|
||||
assert ledger.status(K.REQUIREMENTS) is DeliverableStatus.SUBMITTED
|
||||
assert ledger.revisions(K.REQUIREMENTS) == 2
|
||||
assert ledger.events[-1].type == "escalated"
|
||||
|
||||
|
||||
def test_user_can_override_the_revision_cap():
|
||||
ledger = scoped(max_revisions=1)
|
||||
produce(ledger, K.REQUIREMENTS)
|
||||
ledger.reopen(K.REQUIREMENTS, "first", actor="orchestrator")
|
||||
produce(ledger, K.REQUIREMENTS)
|
||||
|
||||
result = ledger.reopen(
|
||||
K.REQUIREMENTS, "user insists", actor="user", override_cap=True
|
||||
)
|
||||
|
||||
assert result.escalation is None
|
||||
assert ledger.revisions(K.REQUIREMENTS) == 2
|
||||
|
||||
|
||||
def test_default_revision_cap_is_three():
|
||||
assert scoped().max_revisions == 3
|
||||
|
||||
|
||||
# --- records and persistence -------------------------------------------------------
|
||||
|
||||
|
||||
def test_transitions_are_recorded_as_ordered_events():
|
||||
ledger = Ledger(WORKFLOW)
|
||||
produce(ledger, K.SCOPE)
|
||||
|
||||
assert [(e.sequence, e.type, e.subject) for e in ledger.events] == [
|
||||
(1, "started", "scope"),
|
||||
(2, "submitted", "scope"),
|
||||
]
|
||||
|
||||
|
||||
def test_round_trips_through_plain_data():
|
||||
ledger = scoped(max_revisions=5)
|
||||
through_phase_one(ledger)
|
||||
produce(ledger, K.PRODUCT_SELECTION)
|
||||
ledger.reopen(K.PRODUCT_SELECTION, "cost", actor="user")
|
||||
|
||||
restored = Ledger.from_dict(WORKFLOW, ledger.to_dict())
|
||||
|
||||
assert restored.to_dict() == ledger.to_dict()
|
||||
assert restored.ready_work() == ledger.ready_work()
|
||||
assert restored.max_revisions == 5
|
||||
assert restored.events == ledger.events
|
||||
|
||||
|
||||
def test_revising_the_target_architecture_returns_the_code_to_draft():
|
||||
ledger = scoped()
|
||||
through_phase_three(ledger)
|
||||
|
||||
result = ledger.reopen(
|
||||
K.TARGET_ARCHITECTURE, "NFR-1 has no Check on retention", actor="orchestrator"
|
||||
)
|
||||
|
||||
assert set(result.reset) == {
|
||||
K.TARGET_ARCHITECTURE,
|
||||
K.ARCHITECTURE_DESCRIPTION,
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
K.VALIDATION_RESULTS,
|
||||
}
|
||||
assert set(result.withdrawn_approvals) == {
|
||||
P.SOLUTION_ARCHITECTURE,
|
||||
P.SOLUTION_IMPLEMENTATION,
|
||||
P.SOLUTION_VALIDATION,
|
||||
}
|
||||
# The code cannot be rewritten until the revised design is approved again.
|
||||
produce(ledger, K.TARGET_ARCHITECTURE)
|
||||
assert ledger.ready_work() == ()
|
||||
232
tests/execution/test_lessons.py
Normal file
232
tests/execution/test_lessons.py
Normal file
@@ -0,0 +1,232 @@
|
||||
"""Lesson candidates: what an Execution keeps about what went wrong in it."""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from execution_engine.adk.runtime import RoleSpec
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from execution_engine.states.lessons import refusal
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
WORKFLOW,
|
||||
DeliverableKind,
|
||||
PhaseId,
|
||||
sample_submission,
|
||||
)
|
||||
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
|
||||
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
|
||||
|
||||
K = DeliverableKind
|
||||
P = PhaseId
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec(
|
||||
"orchestrator", "Coordinate.", budget=40, skills=("solution-architecture",)
|
||||
),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
FAILED = {"check": "K-2", "evidence": "The identity is shared with another service."}
|
||||
ADDITION = {
|
||||
"statement": "The service sets a request timeout.",
|
||||
"applies_to": "E-1",
|
||||
"reason": "NFR-1 has no Check on timeouts",
|
||||
}
|
||||
|
||||
|
||||
def delegate(*kinds: DeliverableKind) -> Call:
|
||||
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
||||
|
||||
|
||||
def approve(phase: PhaseId) -> Call:
|
||||
return Call("request_approval", {"phase": phase.value, "summary": "done"})
|
||||
|
||||
|
||||
def specialist(kind: DeliverableKind) -> str:
|
||||
return DEPENDENCY_GRAPH[kind].specialist
|
||||
|
||||
|
||||
def failing_validation() -> Call:
|
||||
call = sample_submission(K.VALIDATION_RESULTS)
|
||||
verdicts = json.loads(call.args["attachments"]["verdicts.json"])
|
||||
verdicts["checks"][1] = {**FAILED, "verdict": "fail"}
|
||||
verdicts["proposed_additions"] = [ADDITION]
|
||||
return Call(
|
||||
call.tool,
|
||||
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
|
||||
)
|
||||
|
||||
|
||||
def turns_with_one_revision() -> dict:
|
||||
once = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
return {
|
||||
**once,
|
||||
"orchestrator": [
|
||||
# The first call is refused: requirements are not done yet.
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION), delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.TARGET_ARCHITECTURE)],
|
||||
[
|
||||
approve(P.SOLUTION_ARCHITECTURE),
|
||||
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
||||
[
|
||||
Call(
|
||||
"reopen",
|
||||
{"kind": "deployment_guidance", "reason": "Fix the ingress block."},
|
||||
),
|
||||
delegate(K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
||||
[approve(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
|
||||
[approve(P.PACKAGING), Call("finish")],
|
||||
],
|
||||
specialist(K.DEPLOYMENT_GUIDANCE): [
|
||||
[
|
||||
Call("submit_deliverable", {"content": "No headings yet."}),
|
||||
sample_submission(K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
||||
],
|
||||
specialist(K.VALIDATION_RESULTS): [
|
||||
[failing_validation()],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
async def run(tmp_path, turns, **options):
|
||||
recorder = InMemoryRecorder()
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=FakeAgentRuntime(turns),
|
||||
roles=ROLES,
|
||||
recorder=recorder,
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
skill_versions={"solution-architecture": "abc123"},
|
||||
**options,
|
||||
)
|
||||
result = await engine.start("Ingestion", pre_approved=True)
|
||||
return result, recorder
|
||||
|
||||
|
||||
def of_type(record: dict, kind: str) -> list[dict]:
|
||||
return [c for c in record["candidates"] if c["type"] == kind]
|
||||
|
||||
|
||||
async def test_a_finished_execution_records_what_went_wrong(tmp_path):
|
||||
result, recorder = await run(tmp_path, turns_with_one_revision())
|
||||
|
||||
assert result.status == "completed"
|
||||
record = recorder.lessons[result.execution_id]
|
||||
|
||||
assert of_type(record, "revision") == [
|
||||
{
|
||||
"type": "revision",
|
||||
"deliverable": "deployment_guidance",
|
||||
"reason": "Fix the ingress block.",
|
||||
"requested_by": "orchestrator",
|
||||
}
|
||||
]
|
||||
assert of_type(record, "failed_check") == [
|
||||
{"type": "failed_check", "attempt": 1, **FAILED}
|
||||
]
|
||||
assert of_type(record, "proposed_addition") == [
|
||||
{"type": "proposed_addition", "attempt": 1, **ADDITION}
|
||||
]
|
||||
refusals = of_type(record, "refusal")
|
||||
assert {(r["role"], r["tool"], r["deliverable"]) for r in refusals} == {
|
||||
("orchestrator", "delegate", None),
|
||||
(
|
||||
specialist(K.DEPLOYMENT_GUIDANCE),
|
||||
"submit_deliverable",
|
||||
"deployment_guidance",
|
||||
),
|
||||
}
|
||||
assert all(r["message"].startswith(("Refused", "Not submitted")) for r in refusals)
|
||||
|
||||
|
||||
async def test_the_record_names_the_execution_and_the_skills_loaded(tmp_path):
|
||||
result, recorder = await run(tmp_path, turns_with_one_revision())
|
||||
|
||||
record = recorder.lessons[result.execution_id]
|
||||
|
||||
assert record["execution_id"] == result.execution_id
|
||||
assert record["status"] == "completed"
|
||||
assert record["role_skills"]["orchestrator"] == ["solution-architecture"]
|
||||
assert record["skill_versions"] == {"solution-architecture": "abc123"}
|
||||
|
||||
|
||||
async def test_the_record_is_also_written_beside_the_deliverables_tree(tmp_path):
|
||||
result, recorder = await run(tmp_path, turns_with_one_revision())
|
||||
|
||||
path = (
|
||||
tmp_path
|
||||
/ "lessons"
|
||||
/ "candidates"
|
||||
/ "unassigned"
|
||||
/ f"{result.execution_id}.json"
|
||||
)
|
||||
|
||||
assert (
|
||||
json.loads(path.read_text(encoding="utf-8"))
|
||||
== (recorder.lessons[result.execution_id])
|
||||
)
|
||||
|
||||
|
||||
async def test_an_execution_that_ends_with_warnings_still_records_lessons(tmp_path):
|
||||
turns = {
|
||||
"orchestrator": [[delegate(K.REQUIREMENTS)], []],
|
||||
specialist(K.REQUIREMENTS): [[]], # stops without submitting
|
||||
}
|
||||
|
||||
result, recorder = await run(tmp_path, turns)
|
||||
|
||||
assert result.status == "completed_with_warnings"
|
||||
record = recorder.lessons[result.execution_id]
|
||||
assert of_type(record, "unfinished") == [
|
||||
{
|
||||
"type": "unfinished",
|
||||
"deliverable": "requirements",
|
||||
"role": specialist(K.REQUIREMENTS),
|
||||
"reason": "stopped",
|
||||
}
|
||||
]
|
||||
# Only the scope had been produced, so only the scope was written.
|
||||
written = {path.name for path in tmp_path.rglob("deliverables/*.md")}
|
||||
assert written == {"scope.md"}
|
||||
|
||||
|
||||
def test_a_cloud_read_refusal_keeps_only_its_leading_words():
|
||||
"""What a cloud-read tool says may describe the user's environment."""
|
||||
item = refusal(
|
||||
"requirements-analyst",
|
||||
"requirements",
|
||||
"compute__list_instances",
|
||||
"cloud_read",
|
||||
"Refused: project acme-prod-1234 does not have the API enabled.",
|
||||
)
|
||||
|
||||
assert item["tool"] == "compute__list_instances"
|
||||
assert item["message"] == "Refused"
|
||||
assert "acme-prod-1234" not in item["message"]
|
||||
|
||||
|
||||
def test_refusal_messages_are_bounded():
|
||||
item = refusal(
|
||||
"validator", "validation_results", "terraform_validate", "general", "x" * 900
|
||||
)
|
||||
|
||||
assert len(item["message"]) == 500
|
||||
105
tests/execution/test_outputs_as_it_goes.py
Normal file
105
tests/execution/test_outputs_as_it_goes.py
Normal file
@@ -0,0 +1,105 @@
|
||||
"""An Execution's files are on disk as soon as each Deliverable is submitted.
|
||||
|
||||
Before this they were written only when an Execution ended, so a process that was
|
||||
killed, or an Execution left waiting for its user, had nothing to show.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import FakeAgentRuntime
|
||||
from execution_engine.adk.runtime import NoDecider, RoleSpec
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.baseline import BaselineError, load_baseline
|
||||
from execution_engine.nodes.execution_nodes import _write_outputs
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from execution_engine.states.ledger import Ledger
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
WORKFLOW,
|
||||
sample_submission,
|
||||
)
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def folder(tmp_path, execution_id: str):
|
||||
return tmp_path / "deliverables" / "cases" / "acme" / "executions" / execution_id
|
||||
|
||||
|
||||
async def test_an_execution_waiting_for_its_user_has_written_what_exists(tmp_path):
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=FakeAgentRuntime(
|
||||
{
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
),
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
|
||||
waiting = await engine.start("Ingest", customer_case="acme", execution_id="v1")
|
||||
|
||||
# It stopped for the first Approval: only the scope has been produced.
|
||||
assert waiting.status == "awaiting_user"
|
||||
out = folder(tmp_path, "v1")
|
||||
manifest = json.loads((out / "execution.json").read_text())
|
||||
assert manifest["status"] == "in_progress"
|
||||
assert set(manifest["deliverables"]) == {"scope"}
|
||||
assert (out / "deliverables" / "scope.md").read_text()
|
||||
assert (out / "scope.json").is_file() and (out / "scope.yaml").is_file()
|
||||
assert manifest["solution_guide"] is None
|
||||
# Unfinished work is never something a Change request can start from.
|
||||
try:
|
||||
load_baseline(tmp_path, "acme", "v1")
|
||||
except BaselineError as refused:
|
||||
assert "did not complete (in_progress)" in str(refused)
|
||||
else:
|
||||
raise AssertionError("an in-progress Execution was accepted as a Baseline")
|
||||
|
||||
|
||||
def test_a_file_an_earlier_write_left_is_removed_with_the_revision(tmp_path):
|
||||
state = {
|
||||
"execution_id": "v1",
|
||||
"customer_case": "acme",
|
||||
"workflow_request": "Ingest",
|
||||
"deliverables": {
|
||||
"deployment_guidance": {
|
||||
"content": "guide",
|
||||
"attachments": {
|
||||
"terraform/main.tf": "one",
|
||||
"terraform/queue.tf": "two",
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
out = folder(tmp_path, "v1")
|
||||
|
||||
_write_outputs(tmp_path, state, "in_progress", Ledger(WORKFLOW))
|
||||
assert (out / "terraform" / "queue.tf").is_file()
|
||||
(out / "notes.txt").write_text("a person's own file")
|
||||
|
||||
# The Revision folded the queue into main.tf.
|
||||
state["deliverables"]["deployment_guidance"]["attachments"] = {
|
||||
"terraform/main.tf": "one and two"
|
||||
}
|
||||
_write_outputs(tmp_path, state, "completed", Ledger(WORKFLOW))
|
||||
|
||||
assert not (out / "terraform" / "queue.tf").exists()
|
||||
assert (out / "terraform" / "main.tf").read_text() == "one and two"
|
||||
# Only files this Execution wrote are ever removed.
|
||||
assert (out / "notes.txt").is_file()
|
||||
assert json.loads((out / "execution.json").read_text())["status"] == "completed"
|
||||
639
tests/execution/test_pillars.py
Normal file
639
tests/execution/test_pillars.py
Normal file
@@ -0,0 +1,639 @@
|
||||
"""The design advisor's fan-out to one advisor per architecture-framework pillar."""
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from execution_engine.adk.runtime import RoleSpec, ToolResult, ToolSpec
|
||||
from execution_engine.tools.toolkit import make_tool_factory
|
||||
from tests.reference_workflow.hooks import PILLAR_ADVISOR
|
||||
from tests.reference_workflow.pillars import CONSULT_TOOL, pillar_consultation_tool
|
||||
from tests.reference_workflow.workflow import WORKFLOW, with_pillars
|
||||
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
|
||||
|
||||
PILLARS = {
|
||||
"security": "waf-security",
|
||||
"reliability": "waf-reliability",
|
||||
"sustainability": "waf-sustainability",
|
||||
}
|
||||
ADVISOR = RoleSpec(PILLAR_ADVISOR, "Advise on one pillar.", budget=5)
|
||||
|
||||
|
||||
def advice(pillar: str) -> list[Call]:
|
||||
return [Call("submit_advice", {"content": f"Use strong defaults for {pillar}."})]
|
||||
|
||||
|
||||
def turn_for_pillar(task):
|
||||
return advice(task.context["pillar"])
|
||||
|
||||
|
||||
async def consult(tool: ToolSpec, **args):
|
||||
return await tool.handler(**args)
|
||||
|
||||
|
||||
async def test_consults_every_pillar_with_only_its_skill():
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
|
||||
|
||||
result = await consult(tool, brief="Event ingestion on serverless compute.")
|
||||
|
||||
assert result.outcome is None
|
||||
for pillar in PILLARS:
|
||||
assert f"## {pillar}" in result.message
|
||||
assert f"Use strong defaults for {pillar}." in result.message
|
||||
runs = {run.task.context["pillar"]: run for run in runtime.runs}
|
||||
assert set(runs) == set(PILLARS)
|
||||
assert (
|
||||
runs["security"].task.context["brief"]
|
||||
== "Event ingestion on serverless compute."
|
||||
)
|
||||
assert runtime.runs[0].tools == ("submit_advice",)
|
||||
|
||||
|
||||
async def test_advisors_are_scoped_to_their_pillar_skill():
|
||||
seen: dict[str, tuple[str, ...]] = {}
|
||||
|
||||
class RecordingRuntime(FakeAgentRuntime):
|
||||
async def run(self, role, task, tools):
|
||||
seen[task.context["pillar"]] = role.skills
|
||||
return await super().run(role, task, tools)
|
||||
|
||||
runtime = RecordingRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
|
||||
await consult(pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b")
|
||||
|
||||
assert seen == {pillar: (skill,) for pillar, skill in PILLARS.items()}
|
||||
|
||||
|
||||
async def test_a_subset_of_pillars_can_be_consulted():
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS)
|
||||
|
||||
result = await consult(tool, brief="b", pillars=["reliability"])
|
||||
|
||||
assert "## reliability" in result.message
|
||||
assert "## security" not in result.message
|
||||
|
||||
|
||||
async def test_unknown_pillars_are_refused():
|
||||
tool = pillar_consultation_tool(FakeAgentRuntime({}), ADVISOR, PILLARS)
|
||||
|
||||
result = await consult(tool, brief="b", pillars=["speed"])
|
||||
|
||||
assert result.message.startswith("Refused")
|
||||
assert "security" in result.message
|
||||
|
||||
|
||||
async def test_an_advisor_that_fails_is_reported_without_losing_the_others():
|
||||
def maybe_fail(task):
|
||||
if task.context["pillar"] == "reliability":
|
||||
return [Call("submit_advice", {"content": ""})]
|
||||
return advice(task.context["pillar"])
|
||||
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [maybe_fail] * 3})
|
||||
|
||||
result = await consult(
|
||||
pillar_consultation_tool(runtime, ADVISOR, PILLARS), brief="b"
|
||||
)
|
||||
|
||||
assert "Use strong defaults for security." in result.message
|
||||
assert "## reliability\n\nNo guidance:" in result.message
|
||||
|
||||
|
||||
async def test_tool_factory_assembles_local_mcp_and_consultation_tools():
|
||||
async def noop(**_):
|
||||
return ToolResult("ok")
|
||||
|
||||
def tool(name, tier="general"):
|
||||
return ToolSpec(name, name, {"type": "object", "properties": {}}, noop, tier)
|
||||
|
||||
class Catalog:
|
||||
def tools_for(self, server_ids):
|
||||
available = {
|
||||
"docs": [tool("docs__search")],
|
||||
"compute": [tool("compute__list_instances", "cloud_read")],
|
||||
}
|
||||
return [t for sid in server_ids for t in available.get(sid, [])]
|
||||
|
||||
advisor = RoleSpec(PILLAR_ADVISOR, "", budget=5, mcp_servers=("docs", "compute"))
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar]})
|
||||
factory = make_tool_factory(
|
||||
with_pillars({"security": "waf-security"}),
|
||||
runtime,
|
||||
{PILLAR_ADVISOR: advisor},
|
||||
local_tools=[tool("mermaid_check"), tool("terraform_plan", "cloud_read")],
|
||||
catalog=Catalog(),
|
||||
)
|
||||
|
||||
diagrammer = RoleSpec("diagrammer", "", budget=5, tools=("mermaid_check",))
|
||||
analyst = RoleSpec("requirements-analyst", "", budget=5, mcp_servers=("compute",))
|
||||
assert [t.name for t in factory(diagrammer, "architecture_diagram", {})] == [
|
||||
"mermaid_check"
|
||||
]
|
||||
assert [t.name for t in factory(analyst, "requirements", {})] == [
|
||||
"compute__list_instances"
|
||||
]
|
||||
|
||||
[consult_tool] = factory(
|
||||
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
|
||||
)
|
||||
await consult_tool.handler(brief="b")
|
||||
assert runtime.runs[0].tools == ("submit_advice", "docs__search")
|
||||
|
||||
|
||||
async def test_tool_factory_gives_the_consultation_only_to_the_design_advisor():
|
||||
roles = {PILLAR_ADVISOR: ADVISOR}
|
||||
factory = make_tool_factory(with_pillars(PILLARS), FakeAgentRuntime({}), roles)
|
||||
|
||||
design = factory(
|
||||
RoleSpec("design-advisor", "", budget=5), "design_recommendations", {}
|
||||
)
|
||||
other = factory(RoleSpec("decomposer", "", budget=5), "technical_decomposition", {})
|
||||
|
||||
assert [tool.name for tool in design] == [CONSULT_TOOL]
|
||||
assert list(other) == []
|
||||
|
||||
|
||||
# --- advice remembered across Revisions (iteration 4, step 3) --------------------------
|
||||
|
||||
|
||||
async def test_naming_pillars_reconsults_only_those_and_returns_the_rest_from_memory():
|
||||
memory: dict = {}
|
||||
first = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
await consult(
|
||||
pillar_consultation_tool(first, ADVISOR, PILLARS, memory=memory), brief="v1"
|
||||
)
|
||||
|
||||
def revised(task):
|
||||
pillar = task.context["pillar"]
|
||||
return [Call("submit_advice", {"content": f"Revised advice for {pillar}."})]
|
||||
|
||||
second = FakeAgentRuntime({PILLAR_ADVISOR: [revised]})
|
||||
tool = pillar_consultation_tool(second, ADVISOR, PILLARS, memory=memory)
|
||||
|
||||
result = await consult(tool, brief="v2", pillars=["reliability"])
|
||||
|
||||
assert [run.task.context["pillar"] for run in second.runs] == ["reliability"]
|
||||
assert "## reliability\n\nRevised advice for reliability." in result.message
|
||||
assert (
|
||||
"## security (from your earlier consultation)\n\n"
|
||||
"Use strong defaults for security." in result.message
|
||||
)
|
||||
assert memory["pillar_advice"]["reliability"] == "Revised advice for reliability."
|
||||
assert set(memory["pillar_advice"]) == set(PILLARS)
|
||||
|
||||
|
||||
async def test_omitting_pillars_consults_every_pillar_afresh():
|
||||
memory = {"pillar_advice": {pillar: "old" for pillar in PILLARS}}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
|
||||
|
||||
result = await consult(tool, brief="v2")
|
||||
|
||||
assert len(runtime.runs) == 3
|
||||
assert "earlier consultation" not in result.message
|
||||
assert "old" not in memory["pillar_advice"].values()
|
||||
|
||||
|
||||
async def test_failed_advice_is_not_remembered():
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [advice("security"), []]})
|
||||
tool = pillar_consultation_tool(
|
||||
runtime, ADVISOR, {"security": "s", "reliability": "r"}, memory=memory
|
||||
)
|
||||
|
||||
result = await consult(tool, brief="b")
|
||||
|
||||
assert "## reliability\n\nNo guidance" in result.message
|
||||
assert set(memory["pillar_advice"]) == {"security"}
|
||||
|
||||
|
||||
async def test_the_fan_out_shares_the_cap_and_a_cap_of_one_does_not_deadlock():
|
||||
import asyncio
|
||||
|
||||
from execution_engine.adk.runtime import RunLimiter
|
||||
|
||||
limiter = RunLimiter(1)
|
||||
in_flight = 0
|
||||
peak = 0
|
||||
|
||||
class SlowRuntime(FakeAgentRuntime):
|
||||
async def run(self, role, task, tools):
|
||||
nonlocal in_flight, peak
|
||||
in_flight += 1
|
||||
peak = max(peak, in_flight)
|
||||
await asyncio.sleep(0.01)
|
||||
try:
|
||||
return await super().run(role, task, tools)
|
||||
finally:
|
||||
in_flight -= 1
|
||||
|
||||
runtime = SlowRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, limiter=limiter)
|
||||
|
||||
async with limiter.slot(): # the design advisor's own run holds the only slot
|
||||
result = await asyncio.wait_for(consult(tool, brief="b"), timeout=5)
|
||||
|
||||
assert peak == 1
|
||||
assert result.message.count("Use strong defaults") == 3
|
||||
|
||||
|
||||
async def test_pillar_advice_survives_a_revision_through_the_engine(tmp_path):
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
DeliverableKind,
|
||||
PhaseId,
|
||||
sample_submission,
|
||||
)
|
||||
|
||||
K, P = DeliverableKind, PhaseId
|
||||
design = DEPENDENCY_GRAPH[K.DESIGN_RECOMMENDATIONS].specialist
|
||||
roles = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
|
||||
PILLAR_ADVISOR: ADVISOR,
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
def delegate(*kinds):
|
||||
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
||||
|
||||
def approve(phase):
|
||||
return Call("request_approval", {"phase": phase.value, "summary": "done"})
|
||||
|
||||
submit = sample_submission(K.DESIGN_RECOMMENDATIONS)
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[
|
||||
Call(
|
||||
"reopen",
|
||||
{"kind": "design_recommendations", "reason": "Tighten IAM."},
|
||||
),
|
||||
delegate(K.DESIGN_RECOMMENDATIONS),
|
||||
],
|
||||
[], # stops here: a pre-approved Execution ends with warnings
|
||||
],
|
||||
DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist: [
|
||||
[sample_submission(K.REQUIREMENTS)]
|
||||
],
|
||||
DEPENDENCY_GRAPH[K.TECHNICAL_DECOMPOSITION].specialist: [
|
||||
[sample_submission(K.TECHNICAL_DECOMPOSITION)]
|
||||
],
|
||||
DEPENDENCY_GRAPH[K.PRODUCT_SELECTION].specialist: [
|
||||
[sample_submission(K.PRODUCT_SELECTION)]
|
||||
],
|
||||
design: [
|
||||
[Call(CONSULT_TOOL, {"brief": "v1"}), submit],
|
||||
[Call(CONSULT_TOOL, {"brief": "v2", "pillars": ["security"]}), submit],
|
||||
],
|
||||
PILLAR_ADVISOR: [turn_for_pillar] * 4,
|
||||
}
|
||||
)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
tool_factory=make_tool_factory(with_pillars(PILLARS), runtime, roles),
|
||||
)
|
||||
|
||||
result = await engine.start("Ingestion", pre_approved=True)
|
||||
|
||||
advisor_runs = [r for r in runtime.runs if r.role == PILLAR_ADVISOR]
|
||||
assert [r.task.context["pillar"] for r in advisor_runs[3:]] == ["security"]
|
||||
second = [r for r in runtime.runs if r.role == design][1]
|
||||
assert "## reliability (from your earlier consultation)" in second.results[0][1]
|
||||
stored = result.state["working"]["design_recommendations"]["pillar_advice"]
|
||||
assert set(stored) == set(PILLARS)
|
||||
|
||||
|
||||
# --- each advisor reviews its pillar's checklist ----------------------------------------
|
||||
|
||||
|
||||
def reviewed(task):
|
||||
pillar = task.context["pillar"]
|
||||
return [
|
||||
Call(
|
||||
"submit_advice",
|
||||
{
|
||||
"content": f"Use strong defaults for {pillar}.",
|
||||
"checklist": [
|
||||
{
|
||||
"item": f"{pillar} item one | with a bar",
|
||||
"status": "addressed",
|
||||
"note": "Covered by the first recommendation.",
|
||||
},
|
||||
{
|
||||
"item": f"{pillar} item two",
|
||||
"status": "not_applicable",
|
||||
"note": "An organisation's process, not a design decision.",
|
||||
},
|
||||
{"item": "malformed", "status": "maybe", "note": "dropped"},
|
||||
],
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
async def test_each_advisors_checklist_review_is_kept_beside_its_advice():
|
||||
from tests.reference_workflow.pillars import CHECKLIST_KEY
|
||||
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
|
||||
|
||||
result = await consult(tool, brief="b")
|
||||
|
||||
# The design advisor is given the advice, not the review: it does not copy it.
|
||||
assert "item one" not in result.message
|
||||
assert set(memory[CHECKLIST_KEY]) == set(PILLARS)
|
||||
review = memory[CHECKLIST_KEY]["security"]
|
||||
assert [entry["status"] for entry in review] == ["addressed", "not_applicable"]
|
||||
|
||||
|
||||
async def test_the_review_is_written_into_the_deliverable_by_code():
|
||||
from execution_engine.adk.runtime import Submitted
|
||||
from execution_engine.tools.execution import specialist_tools
|
||||
from tests.reference_workflow.pillars import REVIEW_HEADING
|
||||
from tests.reference_workflow.workflow import DeliverableKind, sample_submission
|
||||
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [reviewed] * 3})
|
||||
await consult(
|
||||
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
|
||||
)
|
||||
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
|
||||
(submit,) = (
|
||||
tool
|
||||
for tool in specialist_tools(
|
||||
WORKFLOW, kind, (), allow_cloud_read=False, working=memory
|
||||
)
|
||||
if tool.name == "submit_deliverable"
|
||||
)
|
||||
call = sample_submission(kind)
|
||||
|
||||
first = await submit.handler(**call.args)
|
||||
# A Revision submits content that already carries the section.
|
||||
again = await submit.handler(
|
||||
content=first.outcome.content, attachments=call.args["attachments"]
|
||||
)
|
||||
|
||||
assert isinstance(first.outcome, Submitted)
|
||||
content = first.outcome.content
|
||||
assert f"## {REVIEW_HEADING}" in content and "### security" in content
|
||||
assert (
|
||||
"| security item one \\| with a bar | Addressed | Covered by the first "
|
||||
"recommendation. |"
|
||||
) in content
|
||||
assert "| security item two | Does not apply |" in content
|
||||
assert "malformed" not in content
|
||||
assert again.outcome.content.count(f"## {REVIEW_HEADING}") == 1
|
||||
|
||||
|
||||
async def test_with_no_review_nothing_is_added():
|
||||
from execution_engine.tools.execution import specialist_tools
|
||||
from tests.reference_workflow.pillars import REVIEW_HEADING
|
||||
from tests.reference_workflow.workflow import DeliverableKind, sample_submission
|
||||
|
||||
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
|
||||
(submit,) = (
|
||||
tool
|
||||
for tool in specialist_tools(
|
||||
WORKFLOW, kind, (), allow_cloud_read=False, working={}
|
||||
)
|
||||
if tool.name == "submit_deliverable"
|
||||
)
|
||||
|
||||
result = await submit.handler(**sample_submission(kind).args)
|
||||
|
||||
assert REVIEW_HEADING not in result.outcome.content
|
||||
|
||||
|
||||
# --- only the pillars a change touches are consulted again -------------------------------
|
||||
|
||||
|
||||
async def test_a_change_names_the_pillars_it_affects_directly():
|
||||
from execution_engine.adk.fake_runtime import FakeDecider
|
||||
from execution_engine.adk.runtime import Answer
|
||||
from tests.reference_workflow.pillars import pillars_touched
|
||||
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"security": Answer(1.9, 0.9),
|
||||
"reliability": Answer(1.0, 0.6),
|
||||
"sustainability": Answer(0.2, 0.8),
|
||||
}
|
||||
)
|
||||
|
||||
chosen = await pillars_touched(
|
||||
decider, list(PILLARS), "Use customer-managed keys.", scope="e1"
|
||||
)
|
||||
|
||||
state, questions, scope = decider.asked[0]
|
||||
assert chosen == ["security"]
|
||||
assert scope == "e1" and state.endswith("Use customer-managed keys.")
|
||||
assert questions["security"].kind == "score"
|
||||
assert questions["security"].options == (
|
||||
"not at all",
|
||||
"only indirectly",
|
||||
"directly",
|
||||
)
|
||||
|
||||
|
||||
async def test_a_change_that_touches_no_pillar_is_an_answer_and_no_decision_is_not():
|
||||
from execution_engine.adk.fake_runtime import FakeDecider
|
||||
from execution_engine.adk.runtime import Answer, NoDecider
|
||||
from tests.reference_workflow.pillars import pillars_touched
|
||||
|
||||
nothing = FakeDecider(dict.fromkeys(PILLARS, Answer(0.0, 0.99)))
|
||||
|
||||
assert await pillars_touched(nothing, list(PILLARS), "Rename a page.") == []
|
||||
assert await pillars_touched(NoDecider(), list(PILLARS), "Rename a page.") is None
|
||||
assert await pillars_touched(nothing, list(PILLARS), " ") is None
|
||||
|
||||
|
||||
async def test_an_empty_list_of_pillars_consults_none_and_returns_what_was_said():
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3})
|
||||
tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory)
|
||||
await consult(tool, brief="b")
|
||||
|
||||
again = await consult(tool, brief="b", pillars=[])
|
||||
|
||||
assert len(runtime.runs) == 3 # nobody was consulted a second time
|
||||
for pillar in PILLARS:
|
||||
assert f"## {pillar} (from your earlier consultation)" in again.message
|
||||
|
||||
|
||||
# --- what the request does not say is a question for the customer ------------------------
|
||||
|
||||
|
||||
def unsure(question: str | None = "Do you have a central identity provider?"):
|
||||
def turn(task):
|
||||
entry = {
|
||||
"item": "Identity is managed centrally",
|
||||
"status": "needs_customer_input",
|
||||
"note": "The request does not say how staff identities are managed.",
|
||||
}
|
||||
if question is not None:
|
||||
entry["question"] = question
|
||||
first = Call("submit_advice", {"content": "Advice.", "checklist": [entry]})
|
||||
# A second attempt, used only when the first is refused.
|
||||
fixed = Call(
|
||||
"submit_advice",
|
||||
{
|
||||
"content": "Advice.",
|
||||
"checklist": [{**entry, "question": "Who manages staff identities?"}],
|
||||
},
|
||||
)
|
||||
return [first, fixed]
|
||||
|
||||
return turn
|
||||
|
||||
|
||||
async def test_an_item_the_request_cannot_settle_is_kept_with_its_question():
|
||||
from tests.reference_workflow.pillars import CHECKLIST_KEY, customer_questions
|
||||
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [unsure()] * 3})
|
||||
|
||||
await consult(
|
||||
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
|
||||
)
|
||||
|
||||
(entry,) = memory[CHECKLIST_KEY]["security"]
|
||||
assert entry["status"] == "needs_customer_input"
|
||||
assert customer_questions(memory)[0] == {
|
||||
"pillar": "security",
|
||||
"item": "Identity is managed centrally",
|
||||
"question": "Do you have a central identity provider?",
|
||||
}
|
||||
assert len(customer_questions(memory)) == len(PILLARS)
|
||||
|
||||
|
||||
async def test_an_item_for_the_customer_must_carry_the_question_to_ask():
|
||||
from tests.reference_workflow.pillars import CHECKLIST_KEY
|
||||
|
||||
memory: dict = {}
|
||||
runtime = FakeAgentRuntime({PILLAR_ADVISOR: [unsure(question=None)] * 3})
|
||||
|
||||
await consult(
|
||||
pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory), brief="b"
|
||||
)
|
||||
|
||||
refused, accepted = (reply for _, reply in runtime.runs[0].results)
|
||||
assert refused.startswith("Not submitted: an item to confirm with the customer")
|
||||
assert "Identity is managed centrally" in refused
|
||||
(entry,) = memory[CHECKLIST_KEY]["security"]
|
||||
assert entry["question"] == "Who manages staff identities?"
|
||||
assert accepted == "Advice submitted."
|
||||
|
||||
|
||||
def test_the_questions_are_reported_as_missing_information_not_as_faults():
|
||||
from tests.reference_workflow.pillars import (
|
||||
CHECKLIST_KEY,
|
||||
QUESTIONS_HEADING,
|
||||
render_checklist_review,
|
||||
)
|
||||
|
||||
working = {
|
||||
CHECKLIST_KEY: {
|
||||
"security": [
|
||||
{
|
||||
"item": "Scans run in the pipeline",
|
||||
"status": "addressed",
|
||||
"note": "DR-3",
|
||||
},
|
||||
{
|
||||
"item": "Identity is managed centrally",
|
||||
"status": "needs_customer_input",
|
||||
"note": "Not said.",
|
||||
"question": "Do you have a central identity provider?",
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
text = render_checklist_review(working)
|
||||
|
||||
assert "| Identity is managed centrally | To confirm with the customer |" in text
|
||||
assert f"### {QUESTIONS_HEADING}" in text
|
||||
assert "They are not faults in the design" in text
|
||||
assert (
|
||||
"| security | Identity is managed centrally | Do you have a central "
|
||||
"identity provider? |"
|
||||
) in text
|
||||
# With nothing to ask there is no questions section.
|
||||
only_addressed = {
|
||||
CHECKLIST_KEY: {"security": working[CHECKLIST_KEY]["security"][:1]}
|
||||
}
|
||||
assert QUESTIONS_HEADING not in render_checklist_review(only_addressed)
|
||||
|
||||
|
||||
async def test_an_execution_writes_its_questions_for_the_customer(tmp_path):
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.runtime import NoDecider
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from tests.reference_workflow.pillars import CONSULT_TOOL
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
DeliverableKind,
|
||||
sample_submission,
|
||||
)
|
||||
|
||||
kind = DeliverableKind.DESIGN_RECOMMENDATIONS
|
||||
advisor = DEPENDENCY_GRAPH[kind].specialist
|
||||
roles = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
PILLAR_ADVISOR: ADVISOR,
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(k)]]
|
||||
for k, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
turns[advisor] = [[Call(CONSULT_TOOL, {"brief": "b"}), sample_submission(kind)]]
|
||||
turns[PILLAR_ADVISOR] = [unsure()] * 3
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
tool_factory=make_tool_factory(with_pillars(PILLARS), runtime, roles),
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest", customer_case="acme", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
out = next(tmp_path.rglob("execution.json")).parent
|
||||
manifest = json.loads((out / "execution.json").read_text())
|
||||
assert manifest["customer_questions"] == "customer-questions.md"
|
||||
questions = (out / "customer-questions.md").read_text()
|
||||
assert questions.startswith("# Questions for the customer")
|
||||
assert "Do you have a central identity provider?" in questions
|
||||
assert (
|
||||
"### Questions for the customer"
|
||||
in (out / "deliverables" / "design_recommendations.md").read_text()
|
||||
)
|
||||
100
tests/execution/test_presentation.py
Normal file
100
tests/execution/test_presentation.py
Normal file
@@ -0,0 +1,100 @@
|
||||
"""Prompt text for plain clients and the /generate response mapping."""
|
||||
|
||||
import json
|
||||
|
||||
from execution_engine.agent import Interaction
|
||||
from tests.reference_workflow.presentation import describe_prompt, to_generate_response
|
||||
|
||||
|
||||
def test_approval_prompt_explains_the_structured_reply():
|
||||
text = describe_prompt(
|
||||
{
|
||||
"approval": {
|
||||
"phase": "solution_architecture",
|
||||
"summary": "Cloud Run and Pub/Sub",
|
||||
"deliverables": ["product_selection", "target_architecture"],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
reply = {"approval": {"phase": "solution_architecture", "decision": "approve"}}
|
||||
assert json.dumps(reply) in text
|
||||
assert "never approves" in text
|
||||
|
||||
|
||||
def test_questions_are_numbered_with_their_source():
|
||||
text = describe_prompt(
|
||||
{
|
||||
"questions": [
|
||||
{"from": "requirements", "question": "RTO?"},
|
||||
{"from": "orchestrator", "question": "Region?"},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert text.startswith("1. (requirements) RTO?\n2. (orchestrator) Region?")
|
||||
|
||||
|
||||
def test_generate_response_maps_deliverables_onto_legacy_artifacts():
|
||||
requirements = (
|
||||
"# Requirements\n## 2.3. Current state\nThree VMs.\n### Detail\nNginx.\n"
|
||||
"## Dependencies\nNone.\n"
|
||||
)
|
||||
interaction = Interaction(
|
||||
"exec-1",
|
||||
"completed_with_warnings",
|
||||
None,
|
||||
{
|
||||
"session_id": "s-1",
|
||||
"warnings": ["revision limit"],
|
||||
"deliverables": {
|
||||
"requirements": {"content": requirements, "attachments": {}},
|
||||
"deployment_guidance": {
|
||||
"content": "# Guide",
|
||||
"attachments": {"terraform/main.tf": "resource {}"},
|
||||
},
|
||||
},
|
||||
"ledger": {
|
||||
"deliverables": {"requirements": {"revisions": 2}},
|
||||
"phases": {
|
||||
"requirements_discovery": "approved",
|
||||
"solution_architecture": "open",
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
body = to_generate_response(interaction, {})
|
||||
|
||||
assert body["status"] == "completed_with_warnings"
|
||||
assert body["validation_passed"] is False
|
||||
assert body["total_loop_iterations"] == 3
|
||||
assert body["current_phase"] == "solution_architecture"
|
||||
assert body["artifacts"]["source_discovery_doc"] == "Three VMs.\n### Detail\nNginx."
|
||||
assert body["artifacts"]["terraform_code"] == "resource {}"
|
||||
assert body["artifacts"]["solution_guide"] is None
|
||||
assert body["warnings"] == ["revision limit"]
|
||||
|
||||
|
||||
def test_the_design_approval_says_what_the_user_is_committing_to():
|
||||
text = describe_prompt(
|
||||
{
|
||||
"approval": {
|
||||
"phase": "solution_architecture",
|
||||
"summary": "Cloud Run and Pub/Sub",
|
||||
"deliverables": ["target_architecture"],
|
||||
"target_architecture": {
|
||||
"elements": 9,
|
||||
"relationships": 11,
|
||||
"checks": 23,
|
||||
"not_applicable": [
|
||||
{"recommendation": "DR-9", "reason": "Single region."}
|
||||
],
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
assert "9 Elements, 11 Relationships and 23 Checks" in text
|
||||
assert "No infrastructure code is written until you approve it" in text
|
||||
assert "Recommendation DR-9 is treated as not applicable: Single region." in text
|
||||
89
tests/execution/test_recording.py
Normal file
89
tests/execution/test_recording.py
Normal file
@@ -0,0 +1,89 @@
|
||||
"""The database recorder and the configured checkpointer."""
|
||||
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
|
||||
from execution_engine.checkpointing import open_checkpointer
|
||||
from execution_engine.config import get_settings
|
||||
from execution_engine.database import DatabaseManager
|
||||
from execution_engine.recorder import DatabaseRecorder
|
||||
from execution_engine.states.ledger import LedgerEvent
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sqlite_db(tmp_path, monkeypatch):
|
||||
path = tmp_path / "records.db"
|
||||
monkeypatch.setattr(get_settings(), "SQLITE_FALLBACK_DB", str(path))
|
||||
return DatabaseManager(db_url="sqlite:///unused"), path
|
||||
|
||||
|
||||
async def test_database_recorder_writes_events_submissions_and_status(sqlite_db):
|
||||
manager, path = sqlite_db
|
||||
recorder = DatabaseRecorder(manager)
|
||||
|
||||
await recorder.record_events(
|
||||
"exec-1",
|
||||
[
|
||||
LedgerEvent(
|
||||
1, "submitted", "requirements", "requirements-analyst", "attempt 1"
|
||||
)
|
||||
],
|
||||
)
|
||||
await recorder.record_submission(
|
||||
"exec-1", "deployment_guidance", "# Guide", {"terraform/main.tf": "resource {}"}
|
||||
)
|
||||
await recorder.record_status(
|
||||
"exec-1",
|
||||
session_id="session-1",
|
||||
workflow_request="Ingestion",
|
||||
status="awaiting_user",
|
||||
current_phase="solution_architecture",
|
||||
details={"revisions": 2},
|
||||
)
|
||||
|
||||
with sqlite3.connect(path) as conn:
|
||||
logs = conn.execute(
|
||||
"SELECT iteration, review_status, reviewer_agent, feedback "
|
||||
"FROM orchestrator_review_logs WHERE execution_id = 'exec-1'"
|
||||
).fetchall()
|
||||
artifacts = conn.execute(
|
||||
"SELECT artifact_type, file_path FROM adk_artifacts WHERE execution_id = 'exec-1' "
|
||||
"ORDER BY file_path"
|
||||
).fetchall()
|
||||
execution = conn.execute(
|
||||
"SELECT status, current_phase, loop_count FROM workflow_executions "
|
||||
"WHERE execution_id = 'exec-1'"
|
||||
).fetchone()
|
||||
session = conn.execute(
|
||||
"SELECT agent_name, state_data FROM adk_sessions WHERE session_id = 'session-1'"
|
||||
).fetchone()
|
||||
|
||||
assert logs == [(1, "submitted", "requirements-analyst", "requirements: attempt 1")]
|
||||
assert artifacts == [
|
||||
("deployment_guidance", "deliverables/deployment_guidance.md"),
|
||||
("deployment_guidance", "terraform/main.tf"),
|
||||
]
|
||||
assert execution == ("awaiting_user", "solution_architecture", 2)
|
||||
assert session[0] == "solution-architect"
|
||||
assert "exec-1" in session[1]
|
||||
|
||||
|
||||
async def test_checkpointer_follows_settings(tmp_path, monkeypatch):
|
||||
settings = get_settings()
|
||||
|
||||
monkeypatch.setattr(settings, "CHECKPOINTER", "memory")
|
||||
async with open_checkpointer(settings) as saver:
|
||||
assert isinstance(saver, InMemorySaver)
|
||||
|
||||
monkeypatch.setattr(settings, "CHECKPOINTER", "sqlite")
|
||||
monkeypatch.setattr(settings, "CHECKPOINT_DB_PATH", str(tmp_path / "cp.db"))
|
||||
async with open_checkpointer(settings) as saver:
|
||||
assert isinstance(saver, AsyncSqliteSaver)
|
||||
|
||||
monkeypatch.setattr(settings, "CHECKPOINTER", "redis")
|
||||
with pytest.raises(ValueError, match="redis"):
|
||||
async with open_checkpointer(settings):
|
||||
pass
|
||||
888
tests/execution/test_requirements_chain.py
Normal file
888
tests/execution/test_requirements_chain.py
Normal file
@@ -0,0 +1,888 @@
|
||||
"""The Requirements chain: the Target architecture graph and the data files around it.
|
||||
|
||||
ADR-0005: the outcome is understood before code is written, and validation answers
|
||||
the graph's Checks and nothing else.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from execution_engine.adk.runtime import RoleSpec
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from execution_engine.tools.execution import specialist_tools
|
||||
from tests.reference_workflow.chain import unimplementable_checks, validation_outcome
|
||||
from tests.reference_workflow.local import mermaid_check
|
||||
from tests.reference_workflow.target_architecture import (
|
||||
GraphError,
|
||||
chain_gaps,
|
||||
parse_graph,
|
||||
render_mermaid,
|
||||
)
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
SAMPLE_DATA,
|
||||
WORKFLOW,
|
||||
DeliverableKind,
|
||||
PhaseId,
|
||||
check_submission,
|
||||
sample_submission,
|
||||
)
|
||||
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
|
||||
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
|
||||
|
||||
K = DeliverableKind
|
||||
P = PhaseId
|
||||
|
||||
REQUIREMENTS = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
COMPONENTS = json.loads(SAMPLE_DATA["components.json"])
|
||||
PRODUCTS = json.loads(SAMPLE_DATA["products.json"])
|
||||
RECOMMENDATIONS = json.loads(SAMPLE_DATA["recommendations.json"])
|
||||
|
||||
|
||||
def graph_data() -> dict:
|
||||
return json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
|
||||
|
||||
def problems(data: dict) -> tuple[str, ...]:
|
||||
with pytest.raises(GraphError) as raised:
|
||||
parse_graph(json.dumps(data))
|
||||
return raised.value.problems
|
||||
|
||||
|
||||
def gaps(data: dict, **overrides) -> tuple[str, ...]:
|
||||
links = {
|
||||
"requirements": REQUIREMENTS,
|
||||
"components": COMPONENTS,
|
||||
"products": PRODUCTS,
|
||||
"recommendations": RECOMMENDATIONS,
|
||||
**overrides,
|
||||
}
|
||||
return chain_gaps(parse_graph(json.dumps(data)), **links)
|
||||
|
||||
|
||||
def upstream(*kinds: DeliverableKind, **replaced: str) -> dict:
|
||||
"""The Deliverables a submission builds on, from the coherent sample."""
|
||||
built = {}
|
||||
for kind in kinds:
|
||||
args = sample_submission(kind).args
|
||||
attachments = {
|
||||
name: replaced.get(name, text) for name, text in args["attachments"].items()
|
||||
}
|
||||
built[kind.value] = {"content": args["content"], "attachments": attachments}
|
||||
return built
|
||||
|
||||
|
||||
DESIGN = (
|
||||
K.REQUIREMENTS,
|
||||
K.TECHNICAL_DECOMPOSITION,
|
||||
K.PRODUCT_SELECTION,
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
)
|
||||
|
||||
|
||||
# --- the graph is well-formed ----------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_sample_graph_parses_into_elements_relationships_and_checks():
|
||||
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
|
||||
|
||||
assert [element.id for element in graph.elements] == ["E-1", "E-2"]
|
||||
assert graph.elements[1].component is None # a supporting Element
|
||||
assert [(r.source, r.target) for r in graph.relationships] == [("E-1", "E-2")]
|
||||
assert graph.check_ids == ("K-1", "K-2")
|
||||
assert graph.summary() == {
|
||||
"elements": 2,
|
||||
"relationships": 1,
|
||||
"checks": 2,
|
||||
"not_applicable": [],
|
||||
}
|
||||
|
||||
|
||||
def test_a_relationship_must_join_two_elements():
|
||||
data = graph_data()
|
||||
data["relationships"][0]["to"] = "E-9"
|
||||
|
||||
assert problems(data) == (
|
||||
"target-architecture.json: Relationship R-1 to E-9, which is not an Element",
|
||||
)
|
||||
|
||||
|
||||
def test_a_check_must_apply_to_something_in_the_graph():
|
||||
data = graph_data()
|
||||
data["checks"][0]["applies_to"] = "E-9"
|
||||
|
||||
assert problems(data) == (
|
||||
(
|
||||
"target-architecture.json: Check K-1 applies to E-9, "
|
||||
"which is not an Element or Relationship"
|
||||
),
|
||||
"target-architecture.json: Element E-1 has no Check",
|
||||
)
|
||||
|
||||
|
||||
def test_every_element_needs_a_check_and_every_id_is_used_once():
|
||||
data = graph_data()
|
||||
data["checks"] = [data["checks"][0]]
|
||||
data["relationships"][0]["id"] = "E-1"
|
||||
|
||||
assert problems(data) == (
|
||||
"target-architecture.json: id E-1 is used more than once",
|
||||
"target-architecture.json: Element E-2 has no Check",
|
||||
)
|
||||
|
||||
|
||||
def test_a_check_without_a_statement_is_named():
|
||||
data = graph_data()
|
||||
data["checks"][1]["statement"] = " "
|
||||
|
||||
assert (
|
||||
"target-architecture.json.checks[1]: needs a non-empty statement"
|
||||
in problems(data)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["not json", "[]", '{"elements": "many"}'])
|
||||
def test_a_graph_that_is_not_the_right_shape_is_refused(text):
|
||||
with pytest.raises(GraphError):
|
||||
parse_graph(text)
|
||||
|
||||
|
||||
# --- the Requirements chain has no gap -------------------------------------------------
|
||||
|
||||
|
||||
def test_the_sample_chain_is_complete():
|
||||
assert gaps(graph_data()) == ()
|
||||
|
||||
|
||||
def test_a_non_functional_requirement_with_no_check_is_a_gap():
|
||||
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
|
||||
|
||||
assert gaps(graph_data(), requirements=[*REQUIREMENTS, retention]) == (
|
||||
"Requirement NFR-2 has no Check",
|
||||
)
|
||||
|
||||
|
||||
def test_a_functional_requirement_is_carried_by_its_components_element():
|
||||
# FR-1 is the source of no Check, but C-1 serves it and E-1 realises C-1.
|
||||
assert not [gap for gap in gaps(graph_data()) if "FR-1" in gap]
|
||||
|
||||
unserved = [{**COMPONENTS[0], "requirements": ["NFR-1"]}]
|
||||
assert gaps(graph_data(), components=unserved) == ("Requirement FR-1 has no Check",)
|
||||
|
||||
|
||||
def test_a_component_with_no_element_is_a_gap():
|
||||
replay = {"id": "C-2", "name": "Replay service", "requirements": []}
|
||||
|
||||
assert gaps(graph_data(), components=[*COMPONENTS, replay]) == (
|
||||
"Component C-2 has no Element",
|
||||
)
|
||||
|
||||
|
||||
def test_a_selected_product_that_no_element_uses_is_a_gap():
|
||||
armour = [{"component": "C-1", "product": "Example Armor"}]
|
||||
|
||||
assert gaps(graph_data(), products=armour) == (
|
||||
"Product Example Armor selected for C-1 appears on no Element",
|
||||
)
|
||||
|
||||
|
||||
def test_a_recommendation_needs_a_check_or_a_reason_it_does_not_apply():
|
||||
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
|
||||
recommendations = [*RECOMMENDATIONS, cmek]
|
||||
|
||||
assert gaps(graph_data(), recommendations=recommendations) == (
|
||||
"Recommendation DR-2 has no Check",
|
||||
)
|
||||
|
||||
excused = graph_data()
|
||||
excused["not_applicable"] = [
|
||||
{"recommendation": "DR-2", "reason": "No regulated data."}
|
||||
]
|
||||
assert gaps(excused, recommendations=recommendations) == ()
|
||||
assert parse_graph(json.dumps(excused)).summary()["not_applicable"] == [
|
||||
{"recommendation": "DR-2", "reason": "No regulated data."}
|
||||
]
|
||||
|
||||
|
||||
def test_a_check_must_come_from_a_requirement_or_recommendation():
|
||||
data = graph_data()
|
||||
data["checks"][0]["source"] = "my own idea"
|
||||
|
||||
assert gaps(data) == (
|
||||
(
|
||||
"Check K-1 has source my own idea, which is not a requirement or design "
|
||||
"recommendation"
|
||||
),
|
||||
"Requirement NFR-1 has no Check",
|
||||
)
|
||||
|
||||
|
||||
def test_a_target_architecture_with_a_gap_cannot_be_submitted():
|
||||
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
|
||||
built_on = upstream(
|
||||
*DESIGN, **{"requirements.json": json.dumps([*REQUIREMENTS, retention])}
|
||||
)
|
||||
args = sample_submission(K.TARGET_ARCHITECTURE).args
|
||||
|
||||
assert check_submission(
|
||||
K.TARGET_ARCHITECTURE, args["content"], args["attachments"], built_on
|
||||
) == ("Requirement NFR-2 has no Check",)
|
||||
assert (
|
||||
check_submission(
|
||||
K.TARGET_ARCHITECTURE,
|
||||
args["content"],
|
||||
args["attachments"],
|
||||
upstream(*DESIGN),
|
||||
)
|
||||
== ()
|
||||
)
|
||||
|
||||
|
||||
# --- the links before the graph --------------------------------------------------------
|
||||
|
||||
|
||||
def test_requirements_need_ids_and_a_kind():
|
||||
def missing(items) -> tuple[str, ...]:
|
||||
args = sample_submission(K.REQUIREMENTS).args
|
||||
return check_submission(
|
||||
K.REQUIREMENTS, args["content"], {"requirements.json": json.dumps(items)}
|
||||
)
|
||||
|
||||
assert missing(REQUIREMENTS) == ()
|
||||
assert missing([]) == ("requirements.json: lists no requirements",)
|
||||
assert missing([REQUIREMENTS[0], {**REQUIREMENTS[0], "kind": "must"}]) == (
|
||||
"requirements.json: id FR-1 is used more than once",
|
||||
"requirements.json: FR-1 kind must be one of functional, non_functional",
|
||||
)
|
||||
assert missing([{"id": "FR-1", "kind": "functional"}]) == (
|
||||
"requirements.json[0]: needs text",
|
||||
)
|
||||
|
||||
|
||||
def test_every_functional_requirement_is_served_by_a_component():
|
||||
args = sample_submission(K.TECHNICAL_DECOMPOSITION).args
|
||||
|
||||
def missing(components) -> tuple[str, ...]:
|
||||
return check_submission(
|
||||
K.TECHNICAL_DECOMPOSITION,
|
||||
args["content"],
|
||||
{"components.json": json.dumps(components)},
|
||||
upstream(K.REQUIREMENTS),
|
||||
)
|
||||
|
||||
assert missing(COMPONENTS) == ()
|
||||
assert missing([{"id": "C-1", "name": "Ingress", "requirements": ["NFR-9"]}]) == (
|
||||
"components.json: C-1 serves NFR-9, which is not a requirement",
|
||||
"components.json: functional requirement FR-1 is served by no component",
|
||||
)
|
||||
|
||||
|
||||
def test_every_component_gets_a_product_and_recommendations_name_real_components():
|
||||
selection = sample_submission(K.PRODUCT_SELECTION).args
|
||||
advice = sample_submission(K.DESIGN_RECOMMENDATIONS).args
|
||||
built_on = upstream(K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
|
||||
|
||||
assert check_submission(
|
||||
K.PRODUCT_SELECTION,
|
||||
selection["content"],
|
||||
{"products.json": json.dumps([{"component": "C-7", "product": "X"}])},
|
||||
built_on,
|
||||
) == (
|
||||
"products.json: C-7 is not a component",
|
||||
"products.json: component C-1 has no product selected",
|
||||
)
|
||||
wandering = [{**RECOMMENDATIONS[0], "components": ["C-7"]}]
|
||||
assert check_submission(
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
advice["content"],
|
||||
{"recommendations.json": json.dumps(wandering)},
|
||||
built_on,
|
||||
) == ("recommendations.json: DR-1 names C-7, which is not a component",)
|
||||
|
||||
|
||||
# --- code and validation answer the graph ----------------------------------------------
|
||||
|
||||
EVERYTHING_BEFORE_CODE = (*DESIGN, K.TARGET_ARCHITECTURE)
|
||||
|
||||
|
||||
def test_deployment_guidance_must_account_for_every_check():
|
||||
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
|
||||
partial = json.dumps(
|
||||
[
|
||||
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
|
||||
{"check": "K-7", "file": "terraform/elsewhere.tf", "address": "b"},
|
||||
]
|
||||
)
|
||||
|
||||
missing = check_submission(
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
args["content"],
|
||||
{**args["attachments"], "implementation.json": partial},
|
||||
upstream(*EVERYTHING_BEFORE_CODE),
|
||||
)
|
||||
|
||||
assert missing == (
|
||||
"implementation.json: K-7 is not a Check in the Target architecture",
|
||||
"implementation.json: Check K-2 is not accounted for",
|
||||
"implementation.json: K-7 names terraform/elsewhere.tf, which is not attached",
|
||||
)
|
||||
|
||||
|
||||
def test_a_check_that_cannot_be_implemented_is_accounted_for_with_its_reason():
|
||||
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
|
||||
implementation = json.dumps(
|
||||
[
|
||||
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
|
||||
{
|
||||
"check": "K-2",
|
||||
"unimplementable": "example_service_account has no such argument",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
missing = check_submission(
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
args["content"],
|
||||
{**args["attachments"], "implementation.json": implementation},
|
||||
upstream(*EVERYTHING_BEFORE_CODE),
|
||||
)
|
||||
|
||||
assert missing == ()
|
||||
assert unimplementable_checks(implementation) == [
|
||||
{"check": "K-2", "reason": "example_service_account has no such argument"}
|
||||
]
|
||||
assert unimplementable_checks(SAMPLE_DATA["implementation.json"]) == []
|
||||
|
||||
|
||||
def test_an_unimplementable_entry_needs_its_reason():
|
||||
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
|
||||
implementation = json.dumps(
|
||||
[
|
||||
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
|
||||
{"check": "K-2", "unimplementable": ""},
|
||||
]
|
||||
)
|
||||
|
||||
missing = check_submission(
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
args["content"],
|
||||
{**args["attachments"], "implementation.json": implementation},
|
||||
upstream(*EVERYTHING_BEFORE_CODE),
|
||||
)
|
||||
|
||||
assert missing == (
|
||||
"implementation.json[0]: needs unimplementable",
|
||||
"implementation.json: Check K-2 is not accounted for",
|
||||
)
|
||||
|
||||
|
||||
def verdicts(**changes) -> str:
|
||||
data = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
data.update(changes)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
def validation_missing(text: str) -> tuple[str, ...]:
|
||||
args = sample_submission(K.VALIDATION_RESULTS).args
|
||||
return check_submission(
|
||||
K.VALIDATION_RESULTS,
|
||||
args["content"],
|
||||
{"verdicts.json": text},
|
||||
upstream(*EVERYTHING_BEFORE_CODE, K.DEPLOYMENT_GUIDANCE),
|
||||
)
|
||||
|
||||
|
||||
def test_validation_gives_exactly_one_verdict_per_check():
|
||||
one = {"check": "K-1", "verdict": "pass", "evidence": "seen"}
|
||||
invented = {"check": "K-9", "verdict": "fail", "evidence": "No Cloud Armor rule."}
|
||||
|
||||
assert validation_missing(SAMPLE_DATA["verdicts.json"]) == ()
|
||||
assert validation_missing(verdicts(checks=[one, one, invented])) == (
|
||||
"verdicts.json.checks: check K-1 is used more than once",
|
||||
"verdicts.json.checks: K-9 is not a Check in the Target architecture",
|
||||
"verdicts.json.checks: Check K-2 is not accounted for",
|
||||
)
|
||||
|
||||
|
||||
def test_the_validator_cannot_fail_the_code_on_something_the_graph_lacks():
|
||||
"""What it notices beyond the Checks is a proposed addition, never a failure."""
|
||||
addition = {
|
||||
"statement": "Cloud Armor rate limiting is configured.",
|
||||
"applies_to": "E-1",
|
||||
"reason": "NFR-1",
|
||||
}
|
||||
text = verdicts(proposed_additions=[addition])
|
||||
|
||||
assert validation_missing(text) == ()
|
||||
outcome = validation_outcome(text)
|
||||
assert outcome["failed"] == []
|
||||
assert outcome["proposed_additions"] == [addition]
|
||||
|
||||
|
||||
def test_a_verdict_needs_evidence_and_a_known_value_and_the_dry_run_a_result():
|
||||
vague = [
|
||||
{"check": "K-1", "verdict": "probably", "evidence": "x"},
|
||||
{"check": "K-2", "verdict": "pass", "evidence": " "},
|
||||
]
|
||||
|
||||
assert validation_missing(verdicts(dry_run={"verdict": "ok"}, checks=vague)) == (
|
||||
"verdicts.json.dry_run: needs a verdict (pass, fail, not_run) and evidence",
|
||||
"verdicts.json.checks[1]: needs evidence",
|
||||
"verdicts.json.checks: K-1 verdict must be one of pass, fail, not_verifiable",
|
||||
"verdicts.json.checks: Check K-2 is not accounted for",
|
||||
)
|
||||
|
||||
|
||||
def test_a_validation_outcome_separates_faults_in_code_from_gaps_in_the_graph():
|
||||
checks = [
|
||||
{"check": "K-1", "verdict": "fail", "evidence": "min instances is 1"},
|
||||
{"check": "K-2", "verdict": "not_verifiable", "evidence": "No such argument."},
|
||||
]
|
||||
|
||||
outcome = validation_outcome(verdicts(checks=checks))
|
||||
|
||||
assert outcome["failed"] == [{"check": "K-1", "evidence": "min instances is 1"}]
|
||||
assert outcome["not_verifiable"] == [
|
||||
{"check": "K-2", "evidence": "No such argument."}
|
||||
]
|
||||
assert outcome["passed"] == []
|
||||
assert outcome["dry_run"]["verdict"] == "pass"
|
||||
|
||||
|
||||
# --- the diagram is a view of the graph ------------------------------------------------
|
||||
|
||||
|
||||
async def test_the_diagram_shows_every_element_and_relationship_once():
|
||||
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
|
||||
|
||||
diagram = render_mermaid(graph)
|
||||
|
||||
assert diagram == render_mermaid(graph) # deterministic
|
||||
assert diagram.count("Ingress service") == 1
|
||||
assert diagram.count("Ingress identity") == 1
|
||||
assert diagram.count("-->") == 1
|
||||
assert "runs as" in diagram
|
||||
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
|
||||
|
||||
|
||||
async def test_names_with_characters_mermaid_reserves_still_render():
|
||||
data = graph_data()
|
||||
data["elements"][0]["name"] = 'Ingest "edge" [v2] (public) | {x}'
|
||||
data["elements"][0]["id"] = "svc/ingest.v2"
|
||||
data["relationships"][0]["from"] = "svc/ingest.v2"
|
||||
data["checks"][0]["applies_to"] = "svc/ingest.v2"
|
||||
data["relationships"][0]["kind"] = "calls (HTTPS) | signed"
|
||||
|
||||
diagram = render_mermaid(parse_graph(json.dumps(data)))
|
||||
|
||||
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
|
||||
assert "svc/ingest.v2" not in diagram
|
||||
|
||||
|
||||
async def test_the_engine_draws_the_diagram_when_the_target_architecture_is_submitted():
|
||||
submit = specialist_tools(
|
||||
WORKFLOW,
|
||||
K.TARGET_ARCHITECTURE,
|
||||
extra=[],
|
||||
allow_cloud_read=False,
|
||||
upstream=upstream(*DESIGN),
|
||||
)[0]
|
||||
args = sample_submission(K.TARGET_ARCHITECTURE).args
|
||||
|
||||
result = await submit.handler(**args)
|
||||
|
||||
assert result.outcome.attachments["architecture.mmd"] == render_mermaid(
|
||||
parse_graph(SAMPLE_DATA["target-architecture.json"])
|
||||
)
|
||||
|
||||
|
||||
async def test_a_gap_is_refused_to_the_specialist_with_the_missing_link_named():
|
||||
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
|
||||
built_on = upstream(
|
||||
*DESIGN, **{"recommendations.json": json.dumps([*RECOMMENDATIONS, cmek])}
|
||||
)
|
||||
submit = specialist_tools(
|
||||
WORKFLOW,
|
||||
K.TARGET_ARCHITECTURE,
|
||||
extra=[],
|
||||
allow_cloud_read=False,
|
||||
upstream=built_on,
|
||||
)[0]
|
||||
|
||||
result = await submit.handler(**sample_submission(K.TARGET_ARCHITECTURE).args)
|
||||
|
||||
assert result.outcome is None
|
||||
assert result.message == (
|
||||
"Not submitted. The Deliverable is missing: Recommendation DR-2 has no Check"
|
||||
)
|
||||
|
||||
|
||||
# --- through the engine ----------------------------------------------------------------
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def delegate(*kinds: DeliverableKind) -> Call:
|
||||
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
||||
|
||||
|
||||
def approve(phase: PhaseId) -> Call:
|
||||
return Call("request_approval", {"phase": phase.value, "summary": "done"})
|
||||
|
||||
|
||||
def specialist(kind: DeliverableKind) -> str:
|
||||
return DEPENDENCY_GRAPH[kind].specialist
|
||||
|
||||
|
||||
def to_design_approval() -> list[list[Call]]:
|
||||
return [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.TARGET_ARCHITECTURE)],
|
||||
]
|
||||
|
||||
|
||||
def engine_for(tmp_path, turns: dict) -> tuple[ExecutionEngine, FakeAgentRuntime]:
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
)
|
||||
return engine, runtime
|
||||
|
||||
|
||||
async def test_the_user_approves_the_graph_before_any_code_is_written(tmp_path):
|
||||
once = {
|
||||
specialist(kind): [[sample_submission(kind)]]
|
||||
for kind in (*DESIGN, K.TARGET_ARCHITECTURE)
|
||||
}
|
||||
engine, runtime = engine_for(
|
||||
tmp_path,
|
||||
{
|
||||
**once,
|
||||
"orchestrator": [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[approve(P.REQUIREMENTS_DISCOVERY)],
|
||||
[delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.TARGET_ARCHITECTURE)],
|
||||
# Tries to start the code before asking: refused, then asks.
|
||||
[delegate(K.DEPLOYMENT_GUIDANCE), approve(P.SOLUTION_ARCHITECTURE)],
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
paused = await engine.resume(
|
||||
first.execution_id,
|
||||
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
|
||||
)
|
||||
|
||||
assert paused.prompt["approval"]["phase"] == "solution_architecture"
|
||||
assert paused.prompt["approval"]["target_architecture"] == {
|
||||
"elements": 2,
|
||||
"relationships": 1,
|
||||
"checks": 2,
|
||||
"not_applicable": [],
|
||||
}
|
||||
refused = runtime.runs[-1].results[0]
|
||||
assert refused[0] == "delegate" and refused[1].startswith("Refused")
|
||||
assert specialist(K.DEPLOYMENT_GUIDANCE) not in [run.role for run in runtime.runs]
|
||||
|
||||
|
||||
async def test_the_orchestrator_is_told_what_validation_decided(tmp_path):
|
||||
failing = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
failing["checks"][0] = {
|
||||
"check": "K-1",
|
||||
"verdict": "fail",
|
||||
"evidence": "min instances is 1",
|
||||
}
|
||||
failed_validation = sample_submission(K.VALIDATION_RESULTS)
|
||||
failed_validation = Call(
|
||||
failed_validation.tool,
|
||||
{
|
||||
**failed_validation.args,
|
||||
"attachments": {"verdicts.json": json.dumps(failing)},
|
||||
},
|
||||
)
|
||||
once = {
|
||||
specialist(kind): [[sample_submission(kind)]]
|
||||
for kind in (*DESIGN, K.TARGET_ARCHITECTURE, K.ARCHITECTURE_DESCRIPTION)
|
||||
}
|
||||
engine, runtime = engine_for(
|
||||
tmp_path,
|
||||
{
|
||||
**once,
|
||||
"orchestrator": [
|
||||
*to_design_approval(),
|
||||
[
|
||||
approve(P.SOLUTION_ARCHITECTURE),
|
||||
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
||||
[
|
||||
Call(
|
||||
"reopen",
|
||||
{
|
||||
"kind": "deployment_guidance",
|
||||
"reason": "K-1 failed: min instances is 1",
|
||||
},
|
||||
),
|
||||
delegate(K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[],
|
||||
],
|
||||
specialist(K.DEPLOYMENT_GUIDANCE): [
|
||||
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
||||
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
||||
],
|
||||
specialist(K.VALIDATION_RESULTS): [[failed_validation]],
|
||||
},
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
turns = [run for run in runtime.runs if run.role == "orchestrator"]
|
||||
before, after = turns[6].task.context, turns[7].task.context
|
||||
assert before["validation"] is None
|
||||
assert after["validation"]["failed"] == [
|
||||
{"check": "K-1", "evidence": "min instances is 1"}
|
||||
]
|
||||
assert after["validation"]["proposed_additions"] == []
|
||||
# Once the code is reopened those verdicts are about work that no longer exists,
|
||||
# so the next turn must not see them and reopen the same thing again.
|
||||
assert turns[8].task.context["validation"] is None
|
||||
# Revising the code leaves the approved design standing.
|
||||
phases = result.state["ledger"]["phases"]
|
||||
assert phases["solution_architecture"] == "approved"
|
||||
revised = [
|
||||
run for run in runtime.runs if run.role == specialist(K.DEPLOYMENT_GUIDANCE)
|
||||
][1]
|
||||
assert revised.task.context["feedback"] == ["K-1 failed: min instances is 1"]
|
||||
|
||||
|
||||
# --- constraints list their conditions --------------------------------------------------
|
||||
|
||||
|
||||
def test_a_non_functional_requirement_must_list_its_conditions():
|
||||
requirements = [
|
||||
{"id": "FR-1", "kind": "functional", "text": "Accept events."},
|
||||
{"id": "NFR-1", "kind": "non_functional", "text": "Encrypted and retained."},
|
||||
{
|
||||
"id": "NFR-2",
|
||||
"kind": "non_functional",
|
||||
"text": "Available.",
|
||||
"conditions": [],
|
||||
},
|
||||
]
|
||||
args = sample_submission(K.REQUIREMENTS).args
|
||||
|
||||
missing = check_submission(
|
||||
K.REQUIREMENTS,
|
||||
args["content"],
|
||||
{"requirements.json": json.dumps(requirements)},
|
||||
)
|
||||
|
||||
needs = "needs `conditions`:"
|
||||
assert [line.split(" a list")[0] for line in missing] == [
|
||||
f"requirements.json: NFR-1 {needs}",
|
||||
f"requirements.json: NFR-2 {needs}",
|
||||
]
|
||||
|
||||
|
||||
def test_a_recommendation_must_list_its_conditions():
|
||||
recommendations = json.loads(SAMPLE_DATA["recommendations.json"])
|
||||
del recommendations[0]["conditions"]
|
||||
args = sample_submission(K.DESIGN_RECOMMENDATIONS).args
|
||||
|
||||
missing = check_submission(
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
args["content"],
|
||||
{"recommendations.json": json.dumps(recommendations)},
|
||||
)
|
||||
|
||||
needs = "needs `conditions`:"
|
||||
assert [line.split(" a list")[0] for line in missing] == [
|
||||
f"recommendations.json: DR-1 {needs}"
|
||||
]
|
||||
|
||||
|
||||
def test_what_code_cannot_show_is_listed_apart_and_needs_no_check():
|
||||
"""An outcome measured in operation is recorded, and no Check is asked for it."""
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
requirements.append(
|
||||
{
|
||||
"id": "NFR-2",
|
||||
"kind": "non_functional",
|
||||
"text": "Pages load within one second at p95.",
|
||||
"conditions": [],
|
||||
"outside_code": ["Measured p95 page load is under one second."],
|
||||
}
|
||||
)
|
||||
args = sample_submission(K.REQUIREMENTS).args
|
||||
|
||||
accepted = check_submission(
|
||||
K.REQUIREMENTS,
|
||||
args["content"],
|
||||
{"requirements.json": json.dumps(requirements)},
|
||||
)
|
||||
gaps = chain_gaps(
|
||||
parse_graph(SAMPLE_DATA["target-architecture.json"]),
|
||||
requirements=requirements,
|
||||
components=json.loads(SAMPLE_DATA["components.json"]),
|
||||
products=json.loads(SAMPLE_DATA["products.json"]),
|
||||
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
|
||||
)
|
||||
|
||||
assert accepted == ()
|
||||
assert gaps == ()
|
||||
|
||||
|
||||
def test_a_constraint_with_conditions_still_needs_a_check():
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
requirements.append(
|
||||
{
|
||||
"id": "NFR-2",
|
||||
"kind": "non_functional",
|
||||
"text": "Two zones, 99.9%.",
|
||||
"conditions": ["The service runs in two zones."],
|
||||
"outside_code": ["Measured availability is 99.9%."],
|
||||
}
|
||||
)
|
||||
|
||||
gaps = chain_gaps(
|
||||
parse_graph(SAMPLE_DATA["target-architecture.json"]),
|
||||
requirements=requirements,
|
||||
components=json.loads(SAMPLE_DATA["components.json"]),
|
||||
products=json.loads(SAMPLE_DATA["products.json"]),
|
||||
recommendations=json.loads(SAMPLE_DATA["recommendations.json"]),
|
||||
)
|
||||
|
||||
assert gaps == ("Requirement NFR-2 has no Check",)
|
||||
|
||||
|
||||
# --- what the customer stated reaches the requirements (ADR-0008) ------------------------
|
||||
|
||||
|
||||
def scoped_upstream(**scope) -> dict:
|
||||
import json
|
||||
|
||||
return {"scope": {"attachments": {"scope.json": json.dumps(scope)}}}
|
||||
|
||||
|
||||
def requirement_problems(requirements: list[dict], **scope) -> list[str]:
|
||||
import json
|
||||
|
||||
from tests.reference_workflow.chain import check_requirements
|
||||
|
||||
return check_requirements(
|
||||
{"requirements.json": json.dumps(requirements)}, scoped_upstream(**scope)
|
||||
)
|
||||
|
||||
|
||||
STATED = {
|
||||
"stated": {
|
||||
"intent": {
|
||||
"requirements": [{"id": "REQ-01", "statement": "Ready in 10 weeks."}]
|
||||
},
|
||||
"networking": {"topology": "hub_and_spoke"},
|
||||
},
|
||||
"binding": ["networking.topology"],
|
||||
}
|
||||
FUNCTIONAL = {"id": "FR-1", "kind": "functional", "text": "Accept events."}
|
||||
CARRIED = {
|
||||
"id": "NFR-1",
|
||||
"kind": "non_functional",
|
||||
"text": "The network is hub and spoke, ready in ten weeks.",
|
||||
"conditions": ["A hub network is connected to each spoke network."],
|
||||
"from": ["REQ-01", "networking.topology"],
|
||||
}
|
||||
|
||||
|
||||
def test_requirements_that_carry_everything_the_customer_stated_are_accepted():
|
||||
assert requirement_problems([FUNCTIONAL, CARRIED], **STATED) == []
|
||||
|
||||
|
||||
def test_a_customers_own_requirement_must_be_covered_by_name():
|
||||
uncited = {**CARRIED, "from": ["networking.topology"]}
|
||||
|
||||
(problem,) = requirement_problems([FUNCTIONAL, uncited], **STATED)
|
||||
|
||||
assert "the customer's requirement REQ-01 is covered by no requirement" in problem
|
||||
|
||||
|
||||
def test_a_stated_choice_must_be_carried_by_a_requirement():
|
||||
"""Case 08: a stated topology was named in the design and not built, and every
|
||||
Check passed, because nothing asked for it."""
|
||||
uncarried = {**CARRIED, "from": ["REQ-01"]}
|
||||
|
||||
(problem,) = requirement_problems([FUNCTIONAL, uncarried], **STATED)
|
||||
|
||||
assert "the customer stated networking.topology and no requirement carries it" in (
|
||||
problem
|
||||
)
|
||||
|
||||
|
||||
def test_a_stated_choice_is_carried_as_a_constraint_the_design_is_checked_against():
|
||||
as_functional = {**CARRIED, "kind": "functional", "conditions": []}
|
||||
outside_code_only = {
|
||||
**CARRIED,
|
||||
"conditions": [],
|
||||
"outside_code": ["The topology is reviewed by the network team."],
|
||||
}
|
||||
|
||||
for weak in (as_functional, outside_code_only):
|
||||
problems = requirement_problems([FUNCTIONAL, weak], **STATED)
|
||||
assert any(
|
||||
"must be non_functional with at least one entry in `conditions`" in p
|
||||
for p in problems
|
||||
), weak
|
||||
|
||||
|
||||
def test_with_no_scope_or_nothing_binding_requirements_are_as_before():
|
||||
from tests.reference_workflow.chain import check_requirements
|
||||
|
||||
plain = [FUNCTIONAL]
|
||||
import json
|
||||
|
||||
assert check_requirements({"requirements.json": json.dumps(plain)}, None) == []
|
||||
assert requirement_problems(plain, stated={"intent": {"statement": "x"}}) == []
|
||||
|
||||
|
||||
def test_what_was_assumed_for_the_customer_does_not_bind_the_design():
|
||||
from execution_engine.scope import Catalogue, FieldSpec, Scope
|
||||
|
||||
catalogue = Catalogue(
|
||||
1,
|
||||
("workload",),
|
||||
{
|
||||
"net.topology": FieldSpec("net.topology", binds=True),
|
||||
"net.region": FieldSpec("net.region", binds=True),
|
||||
"notes": FieldSpec("notes"),
|
||||
},
|
||||
)
|
||||
scope = Scope(
|
||||
catalogue,
|
||||
"workload",
|
||||
values={"net.topology": "hub", "net.region": "x", "notes": "n"},
|
||||
sources={"net.region": "assumed"},
|
||||
)
|
||||
|
||||
assert scope.for_context()["binding"] == ["net.topology"]
|
||||
813
tests/execution/test_router.py
Normal file
813
tests/execution/test_router.py
Normal file
@@ -0,0 +1,813 @@
|
||||
"""The Orchestrator as a router (ADR-0007): readings, a disposition, then a playbook.
|
||||
|
||||
No language model chooses an action here. Specialists are scripted; the Orchestrator
|
||||
is the Router, with a scripted decider standing in for Jev, or none at all.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime, FakeDecider
|
||||
from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.nodes.router import (
|
||||
REPLY_WRITER,
|
||||
Router,
|
||||
RuntimeReplyWriter,
|
||||
Thresholds,
|
||||
)
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
SAMPLE_DATA,
|
||||
WORKFLOW,
|
||||
DeliverableKind,
|
||||
sample_submission,
|
||||
)
|
||||
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
|
||||
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
|
||||
from tests.scoped import scoped_ledger
|
||||
|
||||
K = DeliverableKind
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused by the router.", budget=40),
|
||||
REPLY_WRITER: RoleSpec(REPLY_WRITER, "Write the reply.", budget=3),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def specialist(kind: DeliverableKind) -> str:
|
||||
return DEPENDENCY_GRAPH[kind].specialist
|
||||
|
||||
|
||||
def once() -> dict:
|
||||
return {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
|
||||
|
||||
def validation(**changes) -> Call:
|
||||
"""A validation submission whose verdicts differ from the all-pass sample."""
|
||||
call = sample_submission(K.VALIDATION_RESULTS)
|
||||
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
verdicts.update(changes)
|
||||
return Call(
|
||||
call.tool,
|
||||
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
|
||||
)
|
||||
|
||||
|
||||
def failing(check: str = "K-1") -> Call:
|
||||
checks = json.loads(SAMPLE_DATA["verdicts.json"])["checks"]
|
||||
for item in checks:
|
||||
if item["check"] == check:
|
||||
item.update(verdict="fail", evidence="min instances is 1")
|
||||
return validation(checks=checks)
|
||||
|
||||
|
||||
def addition(reason: str) -> Call:
|
||||
return validation(
|
||||
proposed_additions=[
|
||||
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": reason}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def engine_for(tmp_path, turns: dict, decider=None, writer: bool = False, **options):
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
router = Router(
|
||||
decider or NoDecider(),
|
||||
RuntimeReplyWriter(runtime, ROLES[REPLY_WRITER]) if writer else None,
|
||||
Thresholds(),
|
||||
)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=router,
|
||||
**options,
|
||||
)
|
||||
return engine, runtime
|
||||
|
||||
|
||||
def actions(result) -> list[str]:
|
||||
return [decision["action"] for decision in result.state["decisions"]]
|
||||
|
||||
|
||||
def reopened(result) -> list[tuple[str, str]]:
|
||||
return [
|
||||
(event["subject"], event["detail"])
|
||||
for event in result.state["ledger"]["events"]
|
||||
if event["type"] == "reopened"
|
||||
]
|
||||
|
||||
|
||||
def approve(phase: str) -> dict:
|
||||
return {"approval": {"phase": phase, "decision": "approve"}}
|
||||
|
||||
|
||||
# --- the playbook on rules alone -------------------------------------------------------
|
||||
|
||||
|
||||
async def test_an_execution_completes_with_no_model_choosing_an_action(tmp_path):
|
||||
engine, runtime = engine_for(tmp_path, once())
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert "orchestrator" not in {run.role for run in runtime.runs}
|
||||
assert actions(result) == [
|
||||
"delegate", # the scope
|
||||
"request_approval",
|
||||
"delegate", # requirements
|
||||
"delegate", # technical decomposition
|
||||
"request_approval",
|
||||
"delegate", # product selection
|
||||
"delegate", # design recommendations
|
||||
"delegate", # Target architecture
|
||||
"request_approval",
|
||||
"delegate", # description and deployment guidance, together
|
||||
"request_approval",
|
||||
"delegate", # validation
|
||||
"proceed",
|
||||
"request_approval",
|
||||
"delegate", # solution guide
|
||||
"request_approval",
|
||||
"finish",
|
||||
]
|
||||
groups = {decision["group"] for decision in result.state["decisions"]}
|
||||
assert groups == {
|
||||
"work is ready",
|
||||
"a Phase is complete",
|
||||
"validation decided",
|
||||
"packaging is approved",
|
||||
}
|
||||
|
||||
|
||||
async def test_every_decision_is_recorded_with_its_turn_group_and_action(tmp_path):
|
||||
engine, _ = engine_for(tmp_path, once())
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
first = result.state["decisions"][0]
|
||||
assert first == {
|
||||
"turn": 1,
|
||||
"group": "work is ready",
|
||||
"action": "delegate",
|
||||
"judged": {},
|
||||
}
|
||||
assert [d["turn"] for d in result.state["decisions"]] == sorted(
|
||||
d["turn"] for d in result.state["decisions"]
|
||||
)
|
||||
|
||||
|
||||
async def test_when_no_group_applies_a_pre_approved_execution_ends_with_warnings(
|
||||
tmp_path,
|
||||
):
|
||||
turns = {specialist(K.REQUIREMENTS): [[]]} # stops without submitting
|
||||
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
# The Specialist's question is relayed, answered automatically, and it stops
|
||||
# again; the router never invents work.
|
||||
assert result.status == "completed_with_warnings"
|
||||
|
||||
|
||||
# --- Approvals: the summary is written from data, and only data approves ----------------
|
||||
|
||||
|
||||
async def test_the_approval_summary_is_assembled_from_the_data(tmp_path):
|
||||
engine, _ = engine_for(tmp_path, once())
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
design = await engine.resume(first.execution_id, approve("requirements_discovery"))
|
||||
|
||||
assert first.prompt["approval"]["summary"] == (
|
||||
"The requirements_discovery Phase is ready: requirements, "
|
||||
"technical_decomposition submitted. 2 requirements."
|
||||
)
|
||||
summary = design.prompt["approval"]["summary"]
|
||||
assert "Target architecture: 2 Elements, 1 Relationships, 2 Checks." in summary
|
||||
assert design.prompt["approval"]["target_architecture"]["checks"] == 2
|
||||
|
||||
|
||||
async def test_a_model_writes_the_summary_but_decides_nothing(tmp_path):
|
||||
turns = {
|
||||
**once(),
|
||||
REPLY_WRITER: [
|
||||
[Call("write_reply", {"text": "The scope, ready."})],
|
||||
[Call("write_reply", {"text": "Two requirements, ready."})],
|
||||
],
|
||||
}
|
||||
engine, runtime = engine_for(tmp_path, turns, writer=True)
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
|
||||
assert first.prompt["approval"]["summary"] == "Two requirements, ready."
|
||||
# The writer ran once for the scope and once for the requirements.
|
||||
writer_run = [run for run in runtime.runs if run.role == REPLY_WRITER][-1]
|
||||
assert writer_run.tools == ("write_reply",)
|
||||
assert writer_run.task.context["facts"]["deliverables"] == [
|
||||
"requirements",
|
||||
"technical_decomposition",
|
||||
]
|
||||
|
||||
|
||||
async def test_a_writer_that_fails_does_not_stop_the_approval(tmp_path):
|
||||
engine, _ = engine_for(tmp_path, once(), writer=True) # no writer turn scripted
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
|
||||
assert first.prompt["approval"]["summary"].startswith(
|
||||
"The requirements_discovery Phase is ready"
|
||||
)
|
||||
|
||||
|
||||
async def test_a_text_reply_never_approves_and_the_approval_is_asked_again(tmp_path):
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"asks_change": Answer(0.03, 0.94),
|
||||
"change_target": Answer("requirements", 0.5),
|
||||
"change_clarity": Answer(0.0, 0.9),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, once(), decider)
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
again = await engine.resume(first.execution_id, {"reply": "Looks good to me."})
|
||||
|
||||
assert again.prompt["approval"]["phase"] == "requirements_discovery"
|
||||
assert again.state["ledger"]["phases"]["requirements_discovery"] != "approved"
|
||||
|
||||
|
||||
# --- validation: the code, the graph, or proceed ----------------------------------------
|
||||
|
||||
|
||||
async def test_a_failed_check_sends_the_code_back_and_leaves_the_design_approved(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once()
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [
|
||||
[failing("K-1")],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
engine, runtime = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert reopened(result) == [
|
||||
("deployment_guidance", "K-1 failed: min instances is 1")
|
||||
]
|
||||
assert "revise_code" in actions(result)
|
||||
architect_runs = [
|
||||
run for run in runtime.runs if run.role == specialist(K.TARGET_ARCHITECTURE)
|
||||
]
|
||||
assert len(architect_runs) == 1
|
||||
|
||||
|
||||
async def test_a_check_the_code_cannot_meet_goes_back_to_the_graph_unvalidated(
|
||||
tmp_path,
|
||||
):
|
||||
"""The code's author found the Check names a setting the resource lacks. No
|
||||
validation is spent on it: the Target architecture is revised first."""
|
||||
cannot = sample_submission(K.DEPLOYMENT_GUIDANCE)
|
||||
implementation = json.loads(SAMPLE_DATA["implementation.json"])
|
||||
implementation[1] = {"check": "K-2", "unimplementable": "no such argument"}
|
||||
turns = once()
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [
|
||||
[
|
||||
Call(
|
||||
cannot.tool,
|
||||
{
|
||||
**cannot.args,
|
||||
"attachments": {
|
||||
**cannot.args["attachments"],
|
||||
"implementation.json": json.dumps(implementation),
|
||||
},
|
||||
},
|
||||
)
|
||||
],
|
||||
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
||||
]
|
||||
engine, runtime = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert reopened(result) == [
|
||||
(
|
||||
"target_architecture",
|
||||
(
|
||||
"Check K-2 cannot be implemented as written (no such argument). "
|
||||
"Rewrite it so it can be met, or remove it with a reason. The same "
|
||||
"fault may be in other Checks: read every Check against each reason "
|
||||
"given here and correct every one it applies to."
|
||||
),
|
||||
)
|
||||
]
|
||||
assert "the code cannot meet a Check" in {
|
||||
decision["group"] for decision in result.state["decisions"]
|
||||
}
|
||||
validations = [
|
||||
run for run in runtime.runs if run.role == specialist(K.VALIDATION_RESULTS)
|
||||
]
|
||||
assert len(validations) == 1
|
||||
|
||||
|
||||
async def test_a_check_that_fails_twice_running_is_sent_back_to_the_graph(tmp_path):
|
||||
"""The code was revised and the same Check failed again: the Check is suspect."""
|
||||
turns = once()
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 3
|
||||
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [
|
||||
[failing("K-1")],
|
||||
[failing("K-1")],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
[first, second] = reopened(result)
|
||||
assert first[0] == "deployment_guidance"
|
||||
assert second[0] == "target_architecture"
|
||||
assert "K-1 failed in two validations in a row" in second[1]
|
||||
assert actions(result).count("revise_code") == 1
|
||||
|
||||
|
||||
async def test_without_jev_a_gap_naming_a_real_requirement_revises_the_graph(tmp_path):
|
||||
turns = once()
|
||||
for kind in (
|
||||
K.TARGET_ARCHITECTURE,
|
||||
K.ARCHITECTURE_DESCRIPTION,
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
):
|
||||
turns[specialist(kind)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [
|
||||
[addition("NFR-1 has no Check on timeouts")],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
[(kind, reason)] = reopened(result)
|
||||
assert kind == "target_architecture"
|
||||
assert reason == (
|
||||
"Add a Check on E-1: A timeout is set. (serves NFR-1 has no Check on timeouts)."
|
||||
)
|
||||
|
||||
|
||||
async def test_without_jev_a_gap_that_names_nothing_real_is_left(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("it would be nice")]]
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert reopened(result) == []
|
||||
|
||||
|
||||
async def test_jev_judges_whether_a_gap_is_worth_a_revision(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("NFR-1")]]
|
||||
immaterial = FakeDecider({"gap_material": Answer(0.04, 0.92)})
|
||||
engine, _ = engine_for(tmp_path, turns, immaterial)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
# The rule alone would have revised the graph; the reading says it is not needed.
|
||||
assert reopened(result) == []
|
||||
judged = next(d for d in result.state["decisions"] if d["action"] == "proceed")
|
||||
assert judged["judged"] == {"gap_material": {"value": 0.04, "confidence": 0.92}}
|
||||
[(state, questions, scope)] = immaterial.asked
|
||||
assert "A timeout is set." in state and scope == result.execution_id
|
||||
assert questions["gap_material"].kind == "noul"
|
||||
|
||||
|
||||
async def test_an_uncertain_gap_is_put_to_the_user_who_decides(tmp_path):
|
||||
turns = once()
|
||||
for kind in (
|
||||
K.TARGET_ARCHITECTURE,
|
||||
K.ARCHITECTURE_DESCRIPTION,
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
):
|
||||
turns[specialist(kind)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [
|
||||
[addition("NFR-1")],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"gap_material": Answer(0.5, 0.0), # the grey zone
|
||||
"wants_revision": Answer(0.96, 0.92),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
|
||||
result = await engine.start("Ingest webhooks")
|
||||
for phase in (
|
||||
"requirements_discovery",
|
||||
"solution_architecture",
|
||||
"solution_implementation",
|
||||
):
|
||||
result = await engine.resume(result.execution_id, approve(phase))
|
||||
|
||||
[question] = result.prompt["questions"]
|
||||
assert (
|
||||
"found gaps in the Target architecture: A timeout is set."
|
||||
in (question["question"])
|
||||
)
|
||||
|
||||
result = await engine.resume(result.execution_id, {"reply": "Yes, revise it."})
|
||||
|
||||
assert reopened(result)[0][0] == "target_architecture"
|
||||
# Reopening the graph withdrew the design Approval, so it is asked for again.
|
||||
assert result.prompt["approval"]["phase"] == "solution_architecture"
|
||||
|
||||
|
||||
# --- the user's message: act, queue or close --------------------------------------------
|
||||
|
||||
|
||||
def to_design_approval(engine):
|
||||
async def run():
|
||||
first = await engine.start("Ingest webhooks")
|
||||
return await engine.resume(
|
||||
first.execution_id, approve("requirements_discovery")
|
||||
)
|
||||
|
||||
return run()
|
||||
|
||||
|
||||
def changes(phase: str, feedback: str) -> dict:
|
||||
return {"approval": {"phase": phase, "decision": "changes", "feedback": feedback}}
|
||||
|
||||
|
||||
async def test_a_requested_change_is_routed_to_the_deliverable_it_affects(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"change_target": Answer("design_recommendations", 0.88),
|
||||
"change_clarity": Answer(1.8, 0.8),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
|
||||
design = await to_design_approval(engine)
|
||||
result = await engine.resume(
|
||||
design.execution_id,
|
||||
changes("solution_architecture", "Use customer-managed keys on the topic."),
|
||||
)
|
||||
|
||||
assert reopened(result) == [
|
||||
("design_recommendations", "Use customer-managed keys on the topic.")
|
||||
]
|
||||
# Product selection was not touched; the design is asked for again once redone.
|
||||
assert result.state["ledger"]["deliverables"]["product_selection"]["revisions"] == 0
|
||||
assert result.prompt["approval"]["phase"] == "solution_architecture"
|
||||
[(state, questions, _)] = decider.asked
|
||||
assert "The user's message: Use customer-managed keys on the topic." in state
|
||||
assert questions["change_target"].options == (
|
||||
"scope",
|
||||
"requirements",
|
||||
"technical_decomposition",
|
||||
"product_selection",
|
||||
"design_recommendations",
|
||||
"target_architecture",
|
||||
)
|
||||
assert "asks_change" not in questions # a structured request is known to be one
|
||||
|
||||
|
||||
async def test_an_unclear_change_is_confirmed_with_the_user_before_anything_moves(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once()
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"change_target": Answer("product_selection", 0.31), # not relied on
|
||||
"change_clarity": Answer(1.5, 0.7),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
|
||||
design = await to_design_approval(engine)
|
||||
asked = await engine.resume(
|
||||
design.execution_id, changes("solution_architecture", "Tighten it up.")
|
||||
)
|
||||
|
||||
assert reopened(asked) == []
|
||||
[question] = asked.prompt["questions"]
|
||||
assert question["question"].startswith("Do you want product_selection changed?")
|
||||
|
||||
result = await engine.resume(
|
||||
asked.execution_id,
|
||||
{"reply": "No, change the target architecture: add a Check on timeouts."},
|
||||
)
|
||||
|
||||
[(kind, reason)] = reopened(result)
|
||||
assert kind == "target_architecture"
|
||||
assert "Tighten it up." in reason and "add a Check on timeouts" in reason
|
||||
|
||||
|
||||
async def test_a_free_text_reply_that_asks_for_a_change_is_acted_on(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.REQUIREMENTS)] = [
|
||||
[sample_submission(K.REQUIREMENTS)],
|
||||
[sample_submission(K.REQUIREMENTS)],
|
||||
]
|
||||
turns[specialist(K.TECHNICAL_DECOMPOSITION)] *= 2
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"asks_change": Answer(0.93, 0.86),
|
||||
"change_target": Answer("requirements", 0.9),
|
||||
"change_clarity": Answer(1.9, 0.8),
|
||||
"already_agreed": Answer(0.03, 0.94),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
result = await engine.resume(
|
||||
first.execution_id, {"reply": "Retention must be 7 years, not 30 days."}
|
||||
)
|
||||
|
||||
assert reopened(result) == [
|
||||
("requirements", "Retention must be 7 years, not 30 days.")
|
||||
]
|
||||
|
||||
|
||||
async def test_without_jev_a_change_is_acted_on_only_when_the_deliverable_is_named(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once()
|
||||
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
|
||||
design = await to_design_approval(engine)
|
||||
unnamed = await engine.resume(
|
||||
design.execution_id, changes("solution_architecture", "Make it cheaper.")
|
||||
)
|
||||
|
||||
assert reopened(unnamed) == []
|
||||
assert unnamed.prompt["questions"][0]["question"].startswith(
|
||||
"Which Deliverable should change"
|
||||
)
|
||||
|
||||
named = await engine.resume(
|
||||
unnamed.execution_id, {"reply": "The design recommendations."}
|
||||
)
|
||||
|
||||
[(kind, reason)] = reopened(named)
|
||||
assert kind == "design_recommendations"
|
||||
assert reason == (
|
||||
"Make it cheaper.\nThe user then said: The design recommendations."
|
||||
)
|
||||
|
||||
|
||||
# --- permission is the Ledger's, whatever was decided ------------------------------------
|
||||
|
||||
|
||||
async def test_past_the_revision_limit_the_user_decides(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]]
|
||||
engine, _ = engine_for(tmp_path, turns, max_revisions=1)
|
||||
|
||||
result = await engine.start("Ingest webhooks")
|
||||
for phase in (
|
||||
"requirements_discovery",
|
||||
"solution_architecture",
|
||||
"solution_implementation",
|
||||
"solution_implementation",
|
||||
):
|
||||
result = await engine.resume(result.execution_id, approve(phase))
|
||||
|
||||
assert len(reopened(result)) == 1
|
||||
[question] = result.prompt["questions"]
|
||||
assert (
|
||||
"deployment_guidance has reached its revision limit (1)"
|
||||
in (question["question"])
|
||||
)
|
||||
|
||||
|
||||
# --- the two faults of evaluation case 04 (2026-10-08) -----------------------------------
|
||||
|
||||
|
||||
async def test_a_reply_at_the_revision_limit_is_the_answer_and_the_run_carries_on(
|
||||
tmp_path,
|
||||
):
|
||||
"""The user is told once. Their reply is not read as a new request for changes,
|
||||
the Deliverable is not reopened again, and nothing is asked a second time."""
|
||||
turns = once()
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]]
|
||||
# A decider that would call any reply a clear request to change the code.
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"asks_change": Answer(0.95, 0.9),
|
||||
"change_target": Answer("deployment_guidance", 0.95),
|
||||
"change_clarity": Answer(1.9, 0.8),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider, max_revisions=1)
|
||||
|
||||
result = await engine.start("Ingest webhooks")
|
||||
for phase in (
|
||||
"requirements_discovery",
|
||||
"solution_architecture",
|
||||
"solution_implementation",
|
||||
"solution_implementation",
|
||||
):
|
||||
result = await engine.resume(result.execution_id, approve(phase))
|
||||
[question] = result.prompt["questions"]
|
||||
assert "I will carry on with it as it stands" in question["question"]
|
||||
|
||||
result = await engine.resume(
|
||||
result.execution_id, {"reply": "Please change the code again."}
|
||||
)
|
||||
|
||||
# It moves on to the Approval of validation; the code was reopened once only.
|
||||
assert result.prompt["approval"]["phase"] == "solution_validation"
|
||||
assert len(reopened(result)) == 1
|
||||
assert result.state["revision_limit_asked"] == ["deployment_guidance"]
|
||||
assert decider.asked == [] # the reply was never put to the decider
|
||||
|
||||
|
||||
async def test_only_the_users_own_words_are_read_as_their_message(tmp_path):
|
||||
"""A reply that quotes our question back must not be judged on our words."""
|
||||
question = "Please confirm the requirements draft. Shall I change anything?"
|
||||
turns = once()
|
||||
turns[specialist(K.REQUIREMENTS)] = [
|
||||
[Call("needs_input", {"questions": [question]})],
|
||||
[sample_submission(K.REQUIREMENTS)],
|
||||
]
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"asks_change": Answer(0.05, 0.9),
|
||||
"change_target": Answer("requirements", 0.5),
|
||||
"change_clarity": Answer(0.2, 0.5),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
first = await engine.start("Ingest webhooks")
|
||||
|
||||
result = await engine.resume(
|
||||
first.execution_id, {"reply": f"Q: {question}\nA: Yes, approved."}
|
||||
)
|
||||
# Nothing has been submitted yet, so nothing is judged on that turn. Let the
|
||||
# next reply, to the decomposer's approval prompt, be a text one.
|
||||
result = await engine.resume(result.execution_id, {"reply": "Looks fine."})
|
||||
|
||||
assert result.state["last_reply"] is None
|
||||
state, _, _ = decider.asked[-1]
|
||||
assert state.endswith("The user's message: Looks fine.")
|
||||
|
||||
|
||||
def test_a_quoted_question_is_removed_from_what_the_user_said():
|
||||
from execution_engine.nodes.execution_nodes import _own_words
|
||||
|
||||
asked = ["Please confirm the draft. Shall I change the design?", "Which region?"]
|
||||
reply = (
|
||||
"Q: Please confirm the draft. Shall I change the design?\nA: Yes, approved.\n\n"
|
||||
"Q: Which region?\nA: europe-west2"
|
||||
)
|
||||
|
||||
assert _own_words(reply, asked) == "Yes, approved.\neurope-west2"
|
||||
assert _own_words("Make retention 7 years.", asked) == "Make retention 7 years."
|
||||
|
||||
|
||||
async def test_the_questions_asked_are_given_to_the_decider_as_background(tmp_path):
|
||||
from execution_engine.nodes.router import TurnView
|
||||
|
||||
ledger = scoped_ledger()
|
||||
ledger.start([K.REQUIREMENTS], actor="orchestrator")
|
||||
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"asks_change": Answer(0.05, 0.9),
|
||||
"change_target": Answer("requirements", 0.5),
|
||||
"change_clarity": Answer(0.2, 0.5),
|
||||
}
|
||||
)
|
||||
router = Router(decider, None, Thresholds())
|
||||
view = TurnView(
|
||||
execution_id="e1",
|
||||
pre_approved=False,
|
||||
ledger=ledger,
|
||||
deliverables={},
|
||||
pending_questions=[],
|
||||
outcomes=[],
|
||||
last_reply={
|
||||
"kind": "reply",
|
||||
"text": "Yes, approved.",
|
||||
"asked": ["Shall I change the design?"],
|
||||
},
|
||||
)
|
||||
from execution_engine.nodes.router import Readings
|
||||
|
||||
await router._read_message(view, Readings())
|
||||
|
||||
state, _, _ = decider.asked[0]
|
||||
assert "replying to these questions from the agents:\n- Shall I change" in state
|
||||
assert state.endswith("The user's message: Yes, approved.")
|
||||
|
||||
|
||||
async def test_asking_the_user_again_and_again_with_nothing_done_stops_the_run(
|
||||
tmp_path,
|
||||
):
|
||||
"""Whatever causes it, a run that only asks is stopped, not left to the turn limit."""
|
||||
from execution_engine.nodes.execution_nodes import MAX_IDLE_ASKS
|
||||
from execution_engine.nodes.router import TurnResult
|
||||
|
||||
class OnlyAsks:
|
||||
"""An Orchestrator that asks the user something on every turn."""
|
||||
|
||||
async def turn(self, view, tools):
|
||||
ask = next(tool for tool in tools if tool.name == "ask_user")
|
||||
reply = await ask.handler(questions=["What now?"])
|
||||
return TurnResult(reply.outcome)
|
||||
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=FakeAgentRuntime({}),
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=OnlyAsks(),
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest webhooks")
|
||||
asked = 1
|
||||
while result.status == "awaiting_user" and asked < 20:
|
||||
result = await engine.resume(result.execution_id, {"reply": "Carry on."})
|
||||
asked += 1
|
||||
|
||||
assert result.status == "completed_with_warnings"
|
||||
assert asked == MAX_IDLE_ASKS + 1
|
||||
assert "asked" in result.state["warnings"][-1]
|
||||
assert "no work done in between" in result.state["warnings"][-1]
|
||||
|
||||
|
||||
async def test_a_reply_that_restates_what_is_agreed_is_an_answer_not_a_change(
|
||||
tmp_path,
|
||||
):
|
||||
"""Evaluation case 04: a requirement said again in reply to a later question."""
|
||||
from execution_engine.nodes.router import Readings, TurnView
|
||||
|
||||
ledger = scoped_ledger()
|
||||
ledger.start([K.REQUIREMENTS], actor="orchestrator")
|
||||
ledger.submit(K.REQUIREMENTS, 1, actor="requirements-analyst")
|
||||
view = TurnView(
|
||||
execution_id="e1",
|
||||
pre_approved=False,
|
||||
ledger=ledger,
|
||||
deliverables={
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
||||
}
|
||||
},
|
||||
pending_questions=[],
|
||||
outcomes=[],
|
||||
last_reply={"kind": "reply", "text": "99.9% availability.", "asked": []},
|
||||
)
|
||||
asks = {
|
||||
"asks_change": Answer(0.9, 0.8),
|
||||
"change_target": Answer("requirements", 0.95),
|
||||
"change_clarity": Answer(1.9, 0.8),
|
||||
}
|
||||
|
||||
restated, changed = Readings(), Readings()
|
||||
decider = FakeDecider({**asks, "already_agreed": Answer(0.85, 0.7)})
|
||||
await Router(decider, None, Thresholds())._read_message(view, restated)
|
||||
await Router(
|
||||
FakeDecider({**asks, "already_agreed": Answer(0.02, 0.96)}), None, Thresholds()
|
||||
)._read_message(view, changed)
|
||||
|
||||
assert restated.change == "close"
|
||||
assert changed.change == "act"
|
||||
state, questions, _ = decider.asked[0]
|
||||
assert "The agreed requirements:" in state and "NFR-1" in state
|
||||
assert "already_agreed" in questions
|
||||
126
tests/execution/test_skill_routing.py
Normal file
126
tests/execution/test_skill_routing.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""Which references a Specialist is told to read first (iteration 5 step 8)."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import FakeAgentRuntime, FakeDecider
|
||||
from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from execution_engine.tools.skill_routing import read_first, references_of, routable
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
WORKFLOW,
|
||||
sample_submission,
|
||||
)
|
||||
from tests.reference_workflow.workflow import DeliverableKind as K
|
||||
|
||||
PATTERNS = {
|
||||
"references/cloud-sql.md": "Cloud SQL. A private PostgreSQL instance.",
|
||||
"references/pubsub.md": "Pub/Sub. A topic and a subscription.",
|
||||
}
|
||||
|
||||
|
||||
def skill(root: Path, name: str, references: dict[str, str]) -> Path:
|
||||
folder = root / name
|
||||
(folder / "references").mkdir(parents=True)
|
||||
(folder / "SKILL.md").write_text(f"---\nname: {name}\ndescription: d\n---\n")
|
||||
for file, text in references.items():
|
||||
(folder / "references" / file).write_text(text)
|
||||
return folder
|
||||
|
||||
|
||||
def test_a_reference_is_described_by_its_title_and_first_paragraph(tmp_path):
|
||||
folder = skill(
|
||||
tmp_path,
|
||||
"patterns",
|
||||
{"kms.md": "# Cloud KMS\n\nA key ring and a key.\n\n- A rule.\n"},
|
||||
)
|
||||
|
||||
assert references_of(folder) == {
|
||||
"references/kms.md": "Cloud KMS. A key ring and a key"
|
||||
}
|
||||
|
||||
|
||||
def test_only_a_skill_with_enough_references_is_routed_within(tmp_path):
|
||||
many = skill(tmp_path, "many", {f"r{n}.md": f"# R{n}\n\nText.\n" for n in range(4)})
|
||||
few = skill(tmp_path, "few", {"one.md": "# One\n\nText.\n"})
|
||||
none = tmp_path / "none"
|
||||
none.mkdir()
|
||||
|
||||
assert list(routable({"many": many, "few": few, "none": none})) == ["many"]
|
||||
|
||||
|
||||
async def test_the_references_a_task_needs_are_named():
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"patterns/references/cloud-sql.md": Answer(1.8, 0.9),
|
||||
"patterns/references/pubsub.md": Answer(0.1, 0.9),
|
||||
}
|
||||
)
|
||||
|
||||
chosen = await read_first(decider, "Write the code.", {"patterns": PATTERNS})
|
||||
|
||||
assert chosen == [{"skill": "patterns", "resource": "references/cloud-sql.md"}]
|
||||
_, questions, _ = decider.asked[0]
|
||||
assert "Cloud SQL. A private PostgreSQL instance." in (
|
||||
questions["patterns/references/cloud-sql.md"].instructions
|
||||
)
|
||||
|
||||
|
||||
async def test_without_a_decision_nothing_is_named():
|
||||
assert await read_first(NoDecider(), "Write.", {"patterns": PATTERNS}) is None
|
||||
|
||||
|
||||
async def test_a_specialist_with_routable_skills_is_told_where_to_start(tmp_path):
|
||||
roles = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
def recording(kind):
|
||||
def turn(task):
|
||||
seen[kind.value] = task.context["read_first"]
|
||||
return [sample_submission(kind)]
|
||||
|
||||
return turn
|
||||
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"patterns/references/cloud-sql.md": Answer(0.2, 0.9),
|
||||
"patterns/references/pubsub.md": Answer(1.9, 0.9),
|
||||
}
|
||||
)
|
||||
author = DEPENDENCY_GRAPH[K.DEPLOYMENT_GUIDANCE].specialist
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=FakeAgentRuntime(
|
||||
{
|
||||
spec.specialist: [recording(kind)]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
),
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
decider=decider,
|
||||
skill_references={author: {"patterns": PATTERNS}},
|
||||
)
|
||||
|
||||
await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert seen[K.DEPLOYMENT_GUIDANCE.value] == [
|
||||
{"skill": "patterns", "resource": "references/pubsub.md"}
|
||||
]
|
||||
assert seen[K.REQUIREMENTS.value] is None
|
||||
state, _, _ = decider.asked[0]
|
||||
assert state.startswith("Write the deployment guidance Deliverable")
|
||||
assert "Elements of the Target architecture:" in state
|
||||
306
tests/execution/test_validation_in_pieces.py
Normal file
306
tests/execution/test_validation_in_pieces.py
Normal file
@@ -0,0 +1,306 @@
|
||||
"""Two fixes from the run cancelled on 2026-10-07.
|
||||
|
||||
The validator wrote every verdict in one reply and timed out five times; it now
|
||||
records them a few at a time. And two Checks named settings their resource types do
|
||||
not have, each costing a Revision; the design is now checked against the provider's
|
||||
schema, in code, before it is submitted.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from execution_engine.adk.runtime import Submitted, ToolResult
|
||||
from execution_engine.tools.execution import specialist_tools
|
||||
from tests.reference_workflow.local import TerraformRunner
|
||||
from tests.reference_workflow.schema_review import Reviews, SchemaReview, named_settings
|
||||
from tests.reference_workflow.workflow import (
|
||||
SAMPLE_DATA,
|
||||
WORKFLOW,
|
||||
DeliverableKind,
|
||||
sample_submission,
|
||||
)
|
||||
|
||||
K = DeliverableKind
|
||||
CODE = {"terraform/main.tf": "# v1\n", "implementation.json": "[]"}
|
||||
VERDICTS = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
CONTENT = sample_submission(K.VALIDATION_RESULTS).args["content"]
|
||||
|
||||
|
||||
def upstream(code=CODE) -> dict:
|
||||
return {
|
||||
"target_architecture": {
|
||||
"attachments": {
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
}
|
||||
},
|
||||
"deployment_guidance": {"attachments": dict(code)},
|
||||
}
|
||||
|
||||
|
||||
def validator_tools(working: dict, code=CODE) -> dict:
|
||||
return {
|
||||
tool.name: tool
|
||||
for tool in specialist_tools(
|
||||
WORKFLOW,
|
||||
K.VALIDATION_RESULTS,
|
||||
(),
|
||||
allow_cloud_read=False,
|
||||
upstream=upstream(code),
|
||||
working=working,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
# --- verdicts a few at a time ------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_verdicts_are_recorded_a_few_at_a_time_and_attached_on_submission():
|
||||
working: dict = {}
|
||||
tools = validator_tools(working)
|
||||
|
||||
first = await tools["verdict_put"].handler(
|
||||
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
|
||||
)
|
||||
second = await tools["verdict_put"].handler(checks=VERDICTS["checks"][1:])
|
||||
result = await tools["submit_deliverable"].handler(content=CONTENT)
|
||||
|
||||
assert first.message == "Recorded. 1 of 2 Checks have a verdict. Left: K-2."
|
||||
assert second.message == "Recorded. All 2 Checks have a verdict."
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
attached = json.loads(result.outcome.attachments["verdicts.json"])
|
||||
assert [item["check"] for item in attached["checks"]] == ["K-1", "K-2"]
|
||||
assert attached["dry_run"]["verdict"] == "pass"
|
||||
|
||||
|
||||
async def test_a_submission_with_a_check_left_out_names_it():
|
||||
tools = validator_tools({})
|
||||
await tools["verdict_put"].handler(
|
||||
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
|
||||
)
|
||||
|
||||
result = await tools["submit_deliverable"].handler(content=CONTENT)
|
||||
|
||||
assert result.outcome is None
|
||||
assert "Check K-2 is not accounted for" in result.message
|
||||
|
||||
|
||||
async def test_a_verdict_given_again_replaces_the_first_and_additions_are_kept():
|
||||
working: dict = {}
|
||||
tools = validator_tools(working)
|
||||
addition = {
|
||||
"statement": "A timeout is set.",
|
||||
"applies_to": "E-1",
|
||||
"reason": "NFR-1",
|
||||
}
|
||||
await tools["verdict_put"].handler(checks=VERDICTS["checks"])
|
||||
|
||||
await tools["verdict_put"].handler(
|
||||
checks=[{"check": "K-1", "verdict": "fail", "evidence": "min is 1"}],
|
||||
proposed_additions=[addition, addition],
|
||||
)
|
||||
read = await tools["verdict_read"].handler()
|
||||
|
||||
recorded = json.loads(read.message.rsplit("\n", 1)[0])
|
||||
assert [(c["check"], c["verdict"]) for c in recorded["checks"]] == [
|
||||
("K-1", "fail"),
|
||||
("K-2", "pass"),
|
||||
]
|
||||
assert recorded["proposed_additions"] == [addition]
|
||||
assert read.message.endswith("All 2 Checks have a verdict.")
|
||||
|
||||
|
||||
async def test_a_run_cut_short_carries_on_from_what_it_recorded():
|
||||
working: dict = {}
|
||||
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"][:1])
|
||||
|
||||
# The same code, a later attempt: the working memory came back with the Execution.
|
||||
read = await validator_tools(working)["verdict_read"].handler()
|
||||
|
||||
assert "1 of 2 Checks have a verdict. Left: K-2." in read.message
|
||||
|
||||
|
||||
async def test_verdicts_on_code_that_has_changed_are_dropped():
|
||||
working: dict = {}
|
||||
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"])
|
||||
|
||||
revised = {**CODE, "terraform/main.tf": "# v2\n"}
|
||||
read = await validator_tools(working, revised)["verdict_read"].handler()
|
||||
|
||||
assert read.message.startswith("No verdicts recorded yet.")
|
||||
|
||||
|
||||
async def test_verdicts_passed_as_an_attachment_still_work():
|
||||
call = sample_submission(K.VALIDATION_RESULTS)
|
||||
|
||||
result = await validator_tools({})["submit_deliverable"].handler(**call.args)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
|
||||
|
||||
def test_only_the_validator_records_verdicts():
|
||||
assert "verdict_put" in validator_tools({})
|
||||
assert "verdict_put" not in {
|
||||
tool.name
|
||||
for tool in specialist_tools(
|
||||
WORKFLOW, K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
# --- the design against the provider's schema --------------------------------------------
|
||||
|
||||
|
||||
def test_the_settings_a_check_names_are_read_from_its_statement():
|
||||
assert named_settings("`kms_key_name` is set on the topic") == ("kms_key_name",)
|
||||
assert named_settings("template.scaling.min_instance_count is at least 2") == (
|
||||
"template.scaling.min_instance_count",
|
||||
)
|
||||
assert named_settings("startup_cpu_boost is true") == ("startup_cpu_boost",)
|
||||
# A value in backticks is not a setting.
|
||||
assert named_settings("`enable_cdn` is `false`") == ("enable_cdn",)
|
||||
assert named_settings("`deletion_protection` is `true`") == ("deletion_protection",)
|
||||
# Values, roles and plain prose name no setting.
|
||||
assert named_settings("ingress is INGRESS_TRAFFIC_ALL") == ()
|
||||
assert named_settings("roles/run.invoker is granted to allUsers") == ()
|
||||
assert named_settings('the filter contains "oldest_unacked_message_age"') == ()
|
||||
assert named_settings("The service is private.") == ()
|
||||
|
||||
|
||||
SCHEMA = {
|
||||
"cloud_service": (
|
||||
"example/cloud 1.2.3",
|
||||
{
|
||||
"attributes": {"name": {}, "ingress": {}},
|
||||
"block_types": {
|
||||
"template": {
|
||||
"block": {
|
||||
"attributes": {"service_account": {}},
|
||||
"block_types": {
|
||||
"scaling": {
|
||||
"block": {"attributes": {"min_instance_count": {}}}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
),
|
||||
"cloud_job": ("example/cloud 1.2.3", {"attributes": {"name": {}}}),
|
||||
"cloud_service_iam_member": ("example/cloud 1.2.3", {"attributes": {"role": {}}}),
|
||||
}
|
||||
|
||||
|
||||
class Runner(TerraformRunner):
|
||||
def __init__(self, schemas=SCHEMA) -> None:
|
||||
super().__init__()
|
||||
self._given = schemas
|
||||
|
||||
async def _load_schemas(self):
|
||||
return self._given
|
||||
|
||||
|
||||
def graph(checks, resource_types=("cloud_service", "cloud_job")) -> dict[str, str]:
|
||||
elements = [
|
||||
{
|
||||
"id": f"E-{n}",
|
||||
"name": f"Thing {n}",
|
||||
"product": "Example",
|
||||
"resource_type": kind,
|
||||
}
|
||||
for n, kind in enumerate(resource_types, start=1)
|
||||
]
|
||||
data = {
|
||||
"elements": elements,
|
||||
"relationships": [{"id": "R-1", "from": "E-1", "to": "E-2", "kind": "calls"}],
|
||||
"checks": [
|
||||
{"id": f"K-{n}", "applies_to": on, "statement": text, "source": "NFR-1"}
|
||||
for n, (on, text) in enumerate(checks, start=1)
|
||||
],
|
||||
}
|
||||
return {"target-architecture.json": json.dumps(data)}
|
||||
|
||||
|
||||
async def test_a_check_naming_a_setting_its_resource_type_lacks_is_raised():
|
||||
design = graph(
|
||||
[
|
||||
("E-1", "template.scaling.min_instance_count is at least 2"),
|
||||
("E-2", "`min_instance_count` is at least 2"),
|
||||
("E-2", "the name is set"),
|
||||
]
|
||||
)
|
||||
|
||||
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
|
||||
|
||||
assert len(raised) == 1
|
||||
assert raised[0].startswith(
|
||||
"Check K-2: cloud_job has no setting named min_instance_count"
|
||||
)
|
||||
|
||||
|
||||
async def test_a_setting_found_anywhere_in_the_resource_type_is_accepted():
|
||||
design = graph(
|
||||
[
|
||||
("E-1", "service_account is the dedicated identity"),
|
||||
("E-1", "ingress is INTERNAL"),
|
||||
("E-1", "a cloud_service_iam_member grants the invoker role"),
|
||||
("R-1", "any_made_up_setting is checked on a Relationship"),
|
||||
("E-2", "the name is set"),
|
||||
]
|
||||
)
|
||||
|
||||
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) == ()
|
||||
|
||||
|
||||
async def test_a_value_written_with_dots_is_not_read_as_a_setting():
|
||||
"""A database flag or a service name is a value. A path is a setting only when
|
||||
it starts at a name the resource type has."""
|
||||
design = graph(
|
||||
[
|
||||
("E-1", "the flag cloudsql.iam_authentication is on"),
|
||||
("E-1", "audit logs are on for run.googleapis.com"),
|
||||
("E-1", "template.scaling.made_up_count is at least 2"),
|
||||
("E-2", "the name is set"),
|
||||
]
|
||||
)
|
||||
|
||||
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
|
||||
|
||||
assert len(raised) == 1 and "template.scaling.made_up_count" in raised[0]
|
||||
|
||||
|
||||
async def test_a_resource_type_the_provider_lacks_is_raised_with_close_matches():
|
||||
design = graph(
|
||||
[("E-1", "the name is set"), ("E-2", "the name is set")],
|
||||
resource_types=("cloud_servce", "cloud_job"),
|
||||
)
|
||||
|
||||
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
|
||||
|
||||
assert len(raised) == 1
|
||||
assert raised[0].startswith(
|
||||
"Element E-1: cloud_servce is not a resource type of the provider."
|
||||
)
|
||||
assert "Close matches: cloud_service" in raised[0]
|
||||
|
||||
|
||||
async def test_no_schema_and_other_deliverables_raise_nothing():
|
||||
design = graph([("E-1", "made_up_thing is set"), ("E-2", "the name is set")])
|
||||
unavailable = Runner(ToolResult("Schema lookup is unavailable."))
|
||||
|
||||
assert await SchemaReview(unavailable)(K.TARGET_ARCHITECTURE, design, None) == ()
|
||||
assert await SchemaReview(Runner())(K.DEPLOYMENT_GUIDANCE, design, None) == ()
|
||||
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, {}, None) == ()
|
||||
|
||||
|
||||
async def test_several_reviews_raise_together_and_one_failing_loses_nothing():
|
||||
async def one(kind, attachments, upstream, scope=""):
|
||||
return ["first"]
|
||||
|
||||
async def broken(kind, attachments, upstream, scope=""):
|
||||
raise RuntimeError("down")
|
||||
|
||||
async def two(kind, attachments, upstream, scope=""):
|
||||
return ("second",)
|
||||
|
||||
raised = await Reviews([one, broken, two])(K.TARGET_ARCHITECTURE, {}, None)
|
||||
|
||||
assert raised == ("first", "second")
|
||||
314
tests/execution/test_work_breakdown.py
Normal file
314
tests/execution/test_work_breakdown.py
Normal file
@@ -0,0 +1,314 @@
|
||||
"""The work breakdown structure: a plan checked, scheduled and drawn by code."""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from execution_engine.adk.runtime import NoDecider, RoleSpec
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from tests.reference_workflow.work_breakdown import (
|
||||
check_work_breakdown,
|
||||
mermaid,
|
||||
render,
|
||||
schedule,
|
||||
summary,
|
||||
)
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
SAMPLE_DATA,
|
||||
WORKFLOW,
|
||||
sample_submission,
|
||||
)
|
||||
from tests.reference_workflow.workflow import DeliverableKind as K
|
||||
|
||||
GRAPH = SAMPLE_DATA["target-architecture.json"] # Elements E-1 and E-2
|
||||
PLANNER = DEPENDENCY_GRAPH[K.WORK_BREAKDOWN].specialist
|
||||
|
||||
|
||||
def upstream(**delivery) -> dict:
|
||||
scope = {"stated": {"delivery": delivery}, "assumed": []}
|
||||
return {
|
||||
"target_architecture": {"attachments": {"target-architecture.json": GRAPH}},
|
||||
"scope": {"attachments": {"scope.json": json.dumps(scope)}},
|
||||
}
|
||||
|
||||
|
||||
def plan(packages=None, activities=None) -> dict[str, str]:
|
||||
data = json.loads(SAMPLE_DATA["work-breakdown.json"])
|
||||
if packages is not None:
|
||||
data["work_packages"] = packages
|
||||
if activities is not None:
|
||||
data["activities"] = activities
|
||||
return {"work-breakdown.json": json.dumps(data)}
|
||||
|
||||
|
||||
def activity(ident: str, days: float, *after: str, **more) -> dict:
|
||||
return {
|
||||
"id": ident,
|
||||
"work_package": "WP-1",
|
||||
"name": f"Do {ident}",
|
||||
"days": days,
|
||||
"after": list(after),
|
||||
**more,
|
||||
}
|
||||
|
||||
|
||||
PACKAGE = json.loads(SAMPLE_DATA["work-breakdown.json"])["work_packages"][0]
|
||||
|
||||
|
||||
# --- the plan is checked against the design ---------------------------------------------
|
||||
|
||||
|
||||
def test_a_plan_that_covers_the_design_is_accepted():
|
||||
assert check_work_breakdown(plan(), upstream()) == []
|
||||
|
||||
|
||||
def test_every_element_of_the_design_belongs_to_a_work_package():
|
||||
only_one = [{**PACKAGE, "elements": ["E-1"]}]
|
||||
invented = [{**PACKAGE, "elements": ["E-1", "E-2", "E-9"]}]
|
||||
|
||||
(left_out,) = check_work_breakdown(plan(only_one), upstream())
|
||||
(unknown,) = check_work_breakdown(plan(invented), upstream())
|
||||
|
||||
assert "no work package delivers E-2" in left_out
|
||||
assert "WP-1 delivers E-9, which is not an Element" in unknown
|
||||
|
||||
|
||||
def test_a_plan_that_cannot_be_scheduled_is_refused_with_the_reason():
|
||||
def problems(*activities):
|
||||
return check_work_breakdown(plan(activities=list(activities)), upstream())
|
||||
|
||||
assert any(
|
||||
"depend on each other in a circle" in p
|
||||
for p in problems(activity("A-1", 1, "A-2"), activity("A-2", 1, "A-1"))
|
||||
)
|
||||
assert any(
|
||||
"comes after A-9, which is not an activity" in p
|
||||
for p in problems(activity("A-1", 1, "A-9"))
|
||||
)
|
||||
assert any(
|
||||
"needs `days`, a number above 0" in p for p in problems(activity("A-1", 0))
|
||||
)
|
||||
assert any("split anything over" in p for p in problems(activity("A-1", 400)))
|
||||
assert any(
|
||||
"owner must be one of" in p
|
||||
for p in problems(activity("A-1", 1, owner="vendor"))
|
||||
)
|
||||
assert any(
|
||||
"is in WP-7, not a work package" in p
|
||||
for p in problems({**activity("A-1", 1), "work_package": "WP-7"})
|
||||
)
|
||||
|
||||
|
||||
def test_a_work_package_holds_work_or_other_work_packages():
|
||||
parent = {**PACKAGE, "id": "WP-0", "elements": []}
|
||||
child = {**PACKAGE, "parent": "WP-0"}
|
||||
idle = {**PACKAGE, "id": "WP-2", "elements": []}
|
||||
|
||||
assert check_work_breakdown(plan([parent, child]), upstream()) == []
|
||||
assert any(
|
||||
"WP-2 has no activity and no work package beneath it" in p
|
||||
for p in check_work_breakdown(
|
||||
plan([child | {"parent": None}, idle]), upstream()
|
||||
)
|
||||
)
|
||||
assert any(
|
||||
"has parent WP-9, which does not exist" in p
|
||||
for p in check_work_breakdown(plan([{**PACKAGE, "parent": "WP-9"}]), upstream())
|
||||
)
|
||||
|
||||
|
||||
# --- the schedule is computed -----------------------------------------------------------
|
||||
|
||||
|
||||
def test_an_activity_starts_when_what_it_follows_has_finished():
|
||||
placed = schedule(
|
||||
[activity("A-1", 2), activity("A-2", 3, "A-1"), activity("A-3", 1, "A-2")]
|
||||
)
|
||||
|
||||
assert [(p.start, p.finish) for p in placed.values()] == [(0, 2), (2, 5), (5, 6)]
|
||||
assert all(p.critical for p in placed.values())
|
||||
|
||||
|
||||
def test_independent_work_runs_at_once_up_to_the_size_of_the_team():
|
||||
work = [activity("A-1", 4), activity("A-2", 4), activity("A-3", 4)]
|
||||
|
||||
two = schedule(work, team_size=2)
|
||||
three = schedule(work, team_size=3)
|
||||
|
||||
assert max(p.finish for p in two.values()) == 8
|
||||
assert sorted(p.start for p in two.values()) == [0, 0, 4]
|
||||
assert max(p.finish for p in three.values()) == 4
|
||||
|
||||
|
||||
def test_what_the_customer_owns_takes_nobody_from_the_team():
|
||||
work = [
|
||||
activity("A-1", 10, owner="customer"),
|
||||
activity("A-2", 3),
|
||||
activity("A-3", 2, "A-1", "A-2"),
|
||||
]
|
||||
|
||||
placed = schedule(work, team_size=1)
|
||||
|
||||
# The lead time and the team's work run together; the end waits on the longer.
|
||||
assert placed["A-2"].start == 0 and placed["A-3"].start == 10
|
||||
assert placed["A-1"].critical and placed["A-3"].critical
|
||||
assert not placed["A-2"].critical
|
||||
|
||||
|
||||
def test_when_people_are_short_the_longest_chain_goes_first():
|
||||
work = [activity("A-1", 1), activity("A-2", 1), activity("A-3", 5, "A-2")]
|
||||
|
||||
placed = schedule(work, team_size=1)
|
||||
|
||||
assert placed["A-2"].start == 0 and placed["A-1"].start > 0
|
||||
|
||||
|
||||
# --- the views are written from the plan ------------------------------------------------
|
||||
|
||||
|
||||
def test_the_three_views_are_written_from_the_data():
|
||||
text = render(plan(), upstream(team_size=2))
|
||||
|
||||
assert "## WBS dictionary" in text
|
||||
# Acceptance is the Checks on the work package's Elements, read from the design.
|
||||
assert (
|
||||
"| WP-1 | Ingress | The service and its identity. | a running service | E-1, E-2 | K-1, K-2 |"
|
||||
in text
|
||||
)
|
||||
assert (
|
||||
"| A-2 | Deploy the service | WP-1 | 2 | A-1 | Day 1 | Day 3 | team | yes |"
|
||||
in text
|
||||
)
|
||||
assert "## Timeline" in text and "Day 0" in text
|
||||
assert "3 working days, about 0.6 weeks, with a team of 2 and a 5-day week." in text
|
||||
assert "indicative" in text
|
||||
|
||||
|
||||
def test_a_plan_over_the_delivery_window_says_so_and_names_the_critical_path():
|
||||
long = plan(activities=[activity("A-1", 20), activity("A-2", 15, "A-1")])
|
||||
|
||||
over = render(long, upstream(team_size=2, window_weeks=4))
|
||||
inside = render(long, upstream(team_size=2, window_weeks=10))
|
||||
|
||||
assert "**That is 3.0 weeks over the 4-week delivery window.**" in over
|
||||
assert "The critical path is A-1 -> A-2" in over
|
||||
assert "inside the 10-week delivery window, with 3.0 weeks to spare" in inside
|
||||
assert summary(long, upstream(window_weeks=4))["within_window"] is False
|
||||
|
||||
|
||||
def test_an_assumed_team_size_is_said_to_be_assumed():
|
||||
assert "(assumed; the scope gives no team size)" in render(plan(), upstream())
|
||||
|
||||
|
||||
def test_a_customers_lead_time_is_shown_as_theirs():
|
||||
waiting = plan(
|
||||
activities=[
|
||||
activity(
|
||||
"A-1", 30, owner="customer", assumption="The carrier takes six weeks."
|
||||
),
|
||||
activity("A-2", 2, "A-1"),
|
||||
]
|
||||
)
|
||||
|
||||
text = render(waiting, upstream(team_size=2))
|
||||
|
||||
assert "The customer owns these, and the plan waits on them:" in text
|
||||
assert "- A-1 Do A-1: 30 days. The carrier takes six weeks." in text
|
||||
assert "░" in text and "█" in text
|
||||
|
||||
|
||||
def test_calendar_dates_and_a_chart_appear_only_with_a_start_date():
|
||||
dated = upstream(team_size=2, target_start_date="2026-11-02") # a Monday
|
||||
|
||||
assert mermaid(plan(), upstream(team_size=2)) == ""
|
||||
assert "Dates" not in render(plan(), upstream(team_size=2))
|
||||
chart = mermaid(plan(), dated)
|
||||
assert chart.startswith("gantt\n dateFormat YYYY-MM-DD")
|
||||
assert "Deploy the service :crit, A-2, 2026-11-03, 2d" in chart
|
||||
assert "| 2026-11-03 to 2026-11-04 |" in render(plan(), dated)
|
||||
|
||||
|
||||
def test_a_week_of_working_days_skips_the_weekend():
|
||||
week = plan(activities=[activity("A-1", 5), activity("A-2", 1, "A-1")])
|
||||
|
||||
text = render(week, upstream(team_size=1, target_start_date="2026-11-02"))
|
||||
|
||||
# Five working days from Monday end on Friday; the next starts on Monday.
|
||||
assert "| 2026-11-02 to 2026-11-06 |" in text
|
||||
assert "| 2026-11-09 to 2026-11-09 |" in text
|
||||
|
||||
|
||||
# --- in an Execution --------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_an_execution_delivers_a_plan_and_adds_its_timeline_to_the_guide(
|
||||
tmp_path,
|
||||
):
|
||||
roles = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest", customer_case="acme", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
out = next(tmp_path.rglob("execution.json")).parent
|
||||
breakdown = (out / "deliverables" / "work_breakdown.md").read_text()
|
||||
assert "## WBS dictionary" in breakdown and "## Activity sequence" in breakdown
|
||||
assert (out / "work-breakdown.json").is_file()
|
||||
guide = (out / "solution-architecture-guide.md").read_text()
|
||||
assert "## Delivery plan" in guide and "A-2 Deploy the service" in guide
|
||||
manifest = json.loads((out / "execution.json").read_text())
|
||||
assert manifest["delivery_plan"]["working_days"] == 3
|
||||
assert manifest["delivery_plan"]["critical_path"] == ["A-1", "A-2"]
|
||||
# The planner was given the design and the code to plan from, in full.
|
||||
planner = next(run for run in runtime.runs if run.role == PLANNER)
|
||||
given = planner.task.context["dependencies"]
|
||||
assert "content" in given["target_architecture"]
|
||||
assert "content" in given["deployment_guidance"]
|
||||
|
||||
|
||||
async def test_a_plan_that_leaves_an_element_out_is_returned_to_the_planner(tmp_path):
|
||||
from execution_engine.tools.execution import specialist_tools
|
||||
|
||||
(submit,) = (
|
||||
tool
|
||||
for tool in specialist_tools(
|
||||
WORKFLOW, K.WORK_BREAKDOWN, (), allow_cloud_read=False, upstream=upstream()
|
||||
)
|
||||
if tool.name == "submit_deliverable"
|
||||
)
|
||||
partial = Call(
|
||||
"submit_deliverable",
|
||||
{
|
||||
"content": "## Approach\nIn order.\n",
|
||||
"attachments": plan([{**PACKAGE, "elements": ["E-1"]}]),
|
||||
},
|
||||
)
|
||||
|
||||
refused = await submit.handler(**partial.args)
|
||||
|
||||
assert refused.outcome is None
|
||||
assert "no work package delivers E-2" in refused.message
|
||||
490
tests/execution/test_working_copy.py
Normal file
490
tests/execution/test_working_copy.py
Normal file
@@ -0,0 +1,490 @@
|
||||
"""A Specialist's working copy, and what each Specialist is given to read.
|
||||
|
||||
Three things made an Execution slow: a Specialist rewrote a whole Deliverable in one
|
||||
reply, sent every file again on each tool call, and read every earlier Deliverable in
|
||||
full. A working copy and trimmed reading remove each of them.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from execution_engine.adk.runtime import NoDecider, RoleSpec, Submitted, ToolResult
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from execution_engine.tools.execution import specialist_tools
|
||||
from execution_engine.tools.toolkit import make_tool_factory
|
||||
from execution_engine.tools.workspace import file_tools, graph_tools
|
||||
from tests.reference_workflow.hooks import GRAPH_SHAPE
|
||||
from tests.reference_workflow.local import TerraformRunner, local_tools
|
||||
from tests.reference_workflow.target_architecture import parse_graph, render_tables
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
SAMPLE_DATA,
|
||||
WORKFLOW,
|
||||
DeliverableKind,
|
||||
dependencies_for,
|
||||
sample_submission,
|
||||
)
|
||||
|
||||
K = DeliverableKind
|
||||
GRAPH = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
UPSTREAM = {
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
||||
},
|
||||
"technical_decomposition": {
|
||||
"attachments": {"components.json": SAMPLE_DATA["components.json"]}
|
||||
},
|
||||
"product_selection": {
|
||||
"attachments": {"products.json": SAMPLE_DATA["products.json"]}
|
||||
},
|
||||
"design_recommendations": {
|
||||
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
|
||||
},
|
||||
"target_architecture": {
|
||||
"attachments": {
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def tools_of(kind, working, upstream=UPSTREAM, extra=()):
|
||||
return {
|
||||
tool.name: tool
|
||||
for tool in specialist_tools(
|
||||
WORKFLOW,
|
||||
kind,
|
||||
extra,
|
||||
allow_cloud_read=False,
|
||||
upstream=upstream,
|
||||
working=working,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
# --- the Target architecture is written once -------------------------------------------
|
||||
|
||||
|
||||
def test_tables_are_rendered_from_the_graph():
|
||||
tables = render_tables(parse_graph(SAMPLE_DATA["target-architecture.json"]))
|
||||
|
||||
assert "## Elements" in tables and "## Relationships" in tables
|
||||
assert (
|
||||
"| E-1 | Ingress service | Example Run | example_run_service | C-1 |" in tables
|
||||
)
|
||||
assert "| R-1 | Ingress service | runs as | Ingress identity |" in tables
|
||||
assert "| K-1 | E-1 | The service runs at least two instances. | NFR-1 |" in tables
|
||||
|
||||
|
||||
def test_a_table_cell_cannot_break_its_row():
|
||||
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
graph["checks"][0]["statement"] = "a | b\nc"
|
||||
|
||||
assert "| a \\| b c |" in render_tables(parse_graph(json.dumps(graph)))
|
||||
|
||||
|
||||
async def test_the_engine_writes_the_tables_and_the_diagram_on_submission():
|
||||
tools = tools_of(K.TARGET_ARCHITECTURE, {})
|
||||
|
||||
result = await tools["submit_deliverable"].handler(
|
||||
content="## Design rationale\nOne service, one identity.\n",
|
||||
attachments={
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
},
|
||||
)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
assert result.outcome.content.startswith("## Design rationale")
|
||||
assert "## Checks" in result.outcome.content
|
||||
assert result.outcome.attachments["architecture.mmd"].startswith("flowchart")
|
||||
|
||||
|
||||
async def test_the_graph_is_built_a_few_items_at_a_time_and_attached_on_submission():
|
||||
working: dict = {}
|
||||
tools = tools_of(K.TARGET_ARCHITECTURE, working)
|
||||
|
||||
await tools["graph_put"].handler(elements=GRAPH["elements"])
|
||||
await tools["graph_put"].handler(relationships=GRAPH["relationships"])
|
||||
put = await tools["graph_put"].handler(checks=GRAPH["checks"])
|
||||
assert put.message == (
|
||||
"Added 2, replaced 0. The graph has 2 Elements, 1 Relationships and 2 Checks."
|
||||
)
|
||||
|
||||
result = await tools["submit_deliverable"].handler(
|
||||
content="## Design rationale\nReasons.\n"
|
||||
)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
submitted = json.loads(result.outcome.attachments["target-architecture.json"])
|
||||
assert [item["id"] for item in submitted["checks"]] == ["K-1", "K-2"]
|
||||
|
||||
|
||||
async def test_a_revision_amends_the_graph_that_was_submitted():
|
||||
working: dict = {}
|
||||
tools = tools_of(K.TARGET_ARCHITECTURE, working)
|
||||
await tools["submit_deliverable"].handler(
|
||||
content="## Design rationale\nReasons.\n",
|
||||
attachments={
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
},
|
||||
)
|
||||
|
||||
# A later attempt, with the working memory the Execution kept.
|
||||
tools = tools_of(K.TARGET_ARCHITECTURE, working)
|
||||
changed = {**GRAPH["checks"][0], "statement": "min_instance_count is at least 2"}
|
||||
put = await tools["graph_put"].handler(checks=[changed])
|
||||
read = await tools["graph_read"].handler()
|
||||
|
||||
assert put.message.startswith("Added 0, replaced 1.")
|
||||
assert "min_instance_count is at least 2" in read.message
|
||||
assert len(json.loads(read.message)["checks"]) == 2
|
||||
|
||||
|
||||
async def test_graph_items_can_be_removed_and_an_item_without_an_id_is_refused():
|
||||
working: dict = {}
|
||||
tools = {tool.name: tool for tool in graph_tools(working, GRAPH_SHAPE)}
|
||||
await tools["graph_put"].handler(
|
||||
elements=GRAPH["elements"],
|
||||
not_applicable=[{"recommendation": "DR-9", "reason": "Single region."}],
|
||||
)
|
||||
|
||||
removed = await tools["graph_remove"].handler(ids=["E-2", "DR-9"])
|
||||
refused = await tools["graph_put"].handler(checks=[{"statement": "no id"}])
|
||||
|
||||
assert removed.message.startswith("Removed 2.")
|
||||
assert refused.message.startswith("Refused") and "checks[0]" in refused.message
|
||||
assert [e["id"] for e in working["graph"]["elements"]] == ["E-1"]
|
||||
|
||||
|
||||
# --- files are written once and changed in place ----------------------------------------
|
||||
|
||||
|
||||
async def test_a_file_is_written_once_and_changed_in_place():
|
||||
working: dict = {}
|
||||
tools = {tool.name: tool for tool in file_tools(working)}
|
||||
|
||||
await tools["write_file"].handler(
|
||||
path="terraform/main.tf", content="a = 1\nb = 1\n"
|
||||
)
|
||||
edited = await tools["edit_file"].handler(
|
||||
path="terraform/main.tf", old="b = 1", new="b = 2"
|
||||
)
|
||||
listing = await tools["read_file"].handler()
|
||||
|
||||
assert edited.message == "Edited terraform/main.tf."
|
||||
assert working["files"] == {"terraform/main.tf": "a = 1\nb = 2\n"}
|
||||
assert "- terraform/main.tf (2 lines)" in listing.message
|
||||
|
||||
|
||||
async def test_an_edit_that_does_not_match_exactly_once_changes_nothing():
|
||||
working: dict = {"files": {"main.tf": "x = 1\nx = 1\n"}}
|
||||
tools = {tool.name: tool for tool in file_tools(working)}
|
||||
|
||||
twice = await tools["edit_file"].handler(path="main.tf", old="x = 1", new="x = 2")
|
||||
never = await tools["edit_file"].handler(path="main.tf", old="y", new="z")
|
||||
absent = await tools["edit_file"].handler(path="other.tf", old="x", new="y")
|
||||
unsafe = await tools["write_file"].handler(path="../main.tf", content="x")
|
||||
|
||||
assert "it matches 2" in twice.message and "it matches 0" in never.message
|
||||
assert absent.message.startswith("Refused") and unsafe.message.startswith("Refused")
|
||||
assert working["files"] == {"main.tf": "x = 1\nx = 1\n"}
|
||||
|
||||
|
||||
async def test_the_working_copy_is_what_is_submitted():
|
||||
working: dict = {}
|
||||
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
|
||||
for path in ("terraform/main.tf", "terraform/variables.tf"):
|
||||
await tools["write_file"].handler(path=path, content=f"# {path}\n")
|
||||
await tools["write_file"].handler(
|
||||
path="implementation.json", content=SAMPLE_DATA["implementation.json"]
|
||||
)
|
||||
content = sample_submission(K.DEPLOYMENT_GUIDANCE).args["content"]
|
||||
|
||||
result = await tools["submit_deliverable"].handler(content=content)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
assert set(result.outcome.attachments) == {
|
||||
"terraform/main.tf",
|
||||
"terraform/variables.tf",
|
||||
"implementation.json",
|
||||
}
|
||||
|
||||
|
||||
async def test_a_file_passed_on_submission_replaces_its_working_copy():
|
||||
working: dict = {"files": {"terraform/main.tf": "old\n"}}
|
||||
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
|
||||
call = sample_submission(K.DEPLOYMENT_GUIDANCE)
|
||||
|
||||
result = await tools["submit_deliverable"].handler(**call.args)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
assert result.outcome.attachments["terraform/main.tf"] != "old\n"
|
||||
assert working["files"]["terraform/main.tf"] != "old\n"
|
||||
|
||||
|
||||
def test_only_a_specialist_that_owns_files_can_write_them():
|
||||
assert "write_file" in tools_of(K.DEPLOYMENT_GUIDANCE, {})
|
||||
assert "write_file" not in tools_of(K.VALIDATION_RESULTS, {})
|
||||
assert "graph_put" in tools_of(K.TARGET_ARCHITECTURE, {})
|
||||
assert "graph_put" not in tools_of(K.DEPLOYMENT_GUIDANCE, {})
|
||||
|
||||
|
||||
# --- Terraform runs on the working copy -------------------------------------------------
|
||||
|
||||
|
||||
class Runner(TerraformRunner):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.validated: list[dict[str, str]] = []
|
||||
|
||||
async def validate(self, files):
|
||||
self.validated.append(dict(files))
|
||||
return ToolResult("terraform validate passed:\n")
|
||||
|
||||
async def formatted(self, files):
|
||||
return {name: content.strip() + "\n" for name, content in files.items()}
|
||||
|
||||
|
||||
async def test_terraform_checks_take_no_files_when_there_is_a_working_copy():
|
||||
runner = Runner()
|
||||
files = {"terraform/main.tf": "x = 1\n", "implementation.json": "[]"}
|
||||
(validate,) = local_tools(["terraform_validate"], runner, files)
|
||||
|
||||
await validate.handler()
|
||||
|
||||
assert validate.parameters["required"] == []
|
||||
assert "pass no files" in validate.description
|
||||
# Only the Terraform files are checked.
|
||||
assert runner.validated == [{"terraform/main.tf": "x = 1\n"}]
|
||||
|
||||
|
||||
async def test_formatting_changes_the_working_copy_and_returns_only_names():
|
||||
files = {"terraform/main.tf": " x = 1 \n"}
|
||||
(fmt,) = local_tools(["terraform_fmt"], Runner(), files)
|
||||
|
||||
result = await fmt.handler()
|
||||
|
||||
assert files == {"terraform/main.tf": "x = 1\n"}
|
||||
assert result.message == (
|
||||
"terraform fmt reformatted 1 file(s) in your working copy: terraform/main.tf."
|
||||
)
|
||||
|
||||
|
||||
async def test_files_passed_anyway_are_written_to_the_working_copy_first():
|
||||
runner = Runner()
|
||||
files: dict[str, str] = {}
|
||||
(validate,) = local_tools(["terraform_validate"], runner, files)
|
||||
|
||||
await validate.handler(files=[{"path": "terraform/main.tf", "content": "y"}])
|
||||
|
||||
assert files == {"terraform/main.tf": "y"}
|
||||
assert runner.validated == [files]
|
||||
|
||||
|
||||
def roles_with(tools_by_role: dict[str, tuple[str, ...]]) -> dict[str, RoleSpec]:
|
||||
return {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(
|
||||
spec.specialist,
|
||||
"Produce.",
|
||||
budget=15,
|
||||
tools=tools_by_role.get(spec.specialist, ()),
|
||||
)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def test_the_validator_checks_the_submitted_code_without_being_sent_it(tmp_path):
|
||||
runner = Runner()
|
||||
roles = roles_with({"validator": ("terraform_validate",)})
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
turns["validator"] = [
|
||||
[Call("terraform_validate", {}), sample_submission(K.VALIDATION_RESULTS)]
|
||||
]
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
tool_factory=make_tool_factory(
|
||||
WORKFLOW,
|
||||
runtime,
|
||||
roles,
|
||||
local_tools=lambda working_copy: local_tools(
|
||||
["terraform_validate"], runner, working_copy
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert runner.validated == [
|
||||
{
|
||||
"terraform/main.tf": "# terraform/main.tf\n",
|
||||
"terraform/variables.tf": "# terraform/variables.tf\n",
|
||||
}
|
||||
]
|
||||
# The code is the code author's: validation attaches its verdicts and nothing else.
|
||||
validation = result.state["deliverables"]["validation_results"]
|
||||
assert set(validation["attachments"]) == {"verdicts.json"}
|
||||
|
||||
|
||||
# --- each Specialist reads what it works from -------------------------------------------
|
||||
|
||||
|
||||
def submitted() -> dict[str, dict]:
|
||||
return {
|
||||
str(kind): {
|
||||
"content": f"# {kind}\nProse.\n",
|
||||
"attachments": {
|
||||
name: SAMPLE_DATA.get(name, f"# {name}\n") for name in spec.attachments
|
||||
},
|
||||
}
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
|
||||
|
||||
def test_the_code_is_written_from_the_graph_and_the_ids_alone():
|
||||
given = dependencies_for(K.DEPLOYMENT_GUIDANCE, submitted())
|
||||
|
||||
assert given["target_architecture"] == {
|
||||
"attachments": {
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
}
|
||||
}
|
||||
assert all("content" not in deliverable for deliverable in given.values())
|
||||
assert set(given["requirements"]["attachments"]) == {"requirements.json"}
|
||||
|
||||
|
||||
def test_a_direct_dependency_is_read_whole_and_the_rest_as_data():
|
||||
given = dependencies_for(K.TARGET_ARCHITECTURE, submitted())
|
||||
|
||||
assert "content" in given["product_selection"]
|
||||
assert "content" in given["design_recommendations"]
|
||||
assert "content" not in given["requirements"]
|
||||
assert "content" not in given["technical_decomposition"]
|
||||
|
||||
|
||||
def test_the_validator_reads_the_code_whole_and_the_graph_as_data():
|
||||
given = dependencies_for(K.VALIDATION_RESULTS, submitted())
|
||||
|
||||
assert "terraform/main.tf" in given["deployment_guidance"]["attachments"]
|
||||
assert "content" not in given["target_architecture"]
|
||||
# A Deliverable with no data file has nothing to trim to, so it comes whole.
|
||||
assert "content" in given["architecture_description"]
|
||||
|
||||
|
||||
def test_the_solution_guide_is_assembled_from_every_deliverable_whole():
|
||||
given = dependencies_for(K.SOLUTION_GUIDE, submitted())
|
||||
|
||||
# Every Deliverable but the guide itself and the plan produced beside it.
|
||||
assert len(given) == len(DEPENDENCY_GRAPH) - 2
|
||||
assert all("content" in deliverable for deliverable in given.values())
|
||||
|
||||
|
||||
async def test_a_specialist_is_told_what_its_working_copy_already_holds(tmp_path):
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
failing = sample_submission(K.VALIDATION_RESULTS)
|
||||
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
verdicts["checks"][0].update(verdict="fail", evidence="min instances is 1")
|
||||
turns["validator"] = [
|
||||
[
|
||||
Call(
|
||||
failing.tool,
|
||||
{
|
||||
**failing.args,
|
||||
"attachments": {"verdicts.json": json.dumps(verdicts)},
|
||||
},
|
||||
)
|
||||
],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
turns["iac-author"] = [[sample_submission(K.DEPLOYMENT_GUIDANCE)]] * 2
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=roles_with({}),
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
|
||||
await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
first, second = (run for run in runtime.runs if run.role == "iac-author")
|
||||
assert first.task.context["working_copy"] is None
|
||||
assert second.task.context["working_copy"] == {
|
||||
"files": ["implementation.json", "terraform/main.tf", "terraform/variables.tf"]
|
||||
}
|
||||
|
||||
|
||||
async def test_a_deliverable_given_as_data_can_be_read_whole_on_demand():
|
||||
library = {
|
||||
"requirements": {
|
||||
"content": "# requirements\nProse.\n",
|
||||
"attachments": {"requirements.json": "[]"},
|
||||
}
|
||||
}
|
||||
tools = {
|
||||
tool.name: tool
|
||||
for tool in specialist_tools(
|
||||
WORKFLOW, K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False, library=library
|
||||
)
|
||||
}
|
||||
|
||||
read = await tools["read_deliverable"].handler(kind="requirements")
|
||||
refused = await tools["read_deliverable"].handler(kind="solution_guide")
|
||||
|
||||
assert (
|
||||
read.message.startswith("# requirements")
|
||||
and "requirements.json" in read.message
|
||||
)
|
||||
assert refused.message.startswith("Refused")
|
||||
# With nothing trimmed there is nothing to read back.
|
||||
assert "read_deliverable" not in tools_of(K.REQUIREMENTS, {})
|
||||
|
||||
|
||||
async def test_the_engine_offers_the_trimmed_deliverables_for_reading(tmp_path):
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=roles_with({}),
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
|
||||
await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
author = next(run for run in runtime.runs if run.role == "iac-author")
|
||||
packager = next(run for run in runtime.runs if run.role == "packager")
|
||||
assert "read_deliverable" in author.tools
|
||||
# The author is shown the graph alone; the rest it can read when it needs to.
|
||||
assert list(author.task.context["dependencies"]) == ["target_architecture"]
|
||||
assert "content" not in author.task.context["dependencies"]["target_architecture"]
|
||||
assert "read_deliverable" not in packager.tools
|
||||
Reference in New Issue
Block a user