Files
orchestration_engine/tests/execution/test_working_copy.py

491 lines
18 KiB
Python

"""A Specialist's working copy, and what each Specialist is given to read.
Three things made an Execution slow: a Specialist rewrote a whole Deliverable in one
reply, sent every file again on each tool call, and read every earlier Deliverable in
full. A working copy and trimmed reading remove each of them.
"""
import json
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.runtime import NoDecider, RoleSpec, Submitted, ToolResult
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from execution_engine.tools.execution import specialist_tools
from execution_engine.tools.toolkit import make_tool_factory
from execution_engine.tools.workspace import file_tools, graph_tools
from tests.reference_workflow.hooks import GRAPH_SHAPE
from tests.reference_workflow.local import TerraformRunner, local_tools
from tests.reference_workflow.target_architecture import parse_graph, render_tables
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
SAMPLE_DATA,
WORKFLOW,
DeliverableKind,
dependencies_for,
sample_submission,
)
K = DeliverableKind
GRAPH = json.loads(SAMPLE_DATA["target-architecture.json"])
UPSTREAM = {
"requirements": {
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
},
"technical_decomposition": {
"attachments": {"components.json": SAMPLE_DATA["components.json"]}
},
"product_selection": {
"attachments": {"products.json": SAMPLE_DATA["products.json"]}
},
"design_recommendations": {
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
},
"target_architecture": {
"attachments": {
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
}
},
}
def tools_of(kind, working, upstream=UPSTREAM, extra=()):
return {
tool.name: tool
for tool in specialist_tools(
WORKFLOW,
kind,
extra,
allow_cloud_read=False,
upstream=upstream,
working=working,
)
}
# --- the Target architecture is written once -------------------------------------------
def test_tables_are_rendered_from_the_graph():
tables = render_tables(parse_graph(SAMPLE_DATA["target-architecture.json"]))
assert "## Elements" in tables and "## Relationships" in tables
assert (
"| E-1 | Ingress service | Example Run | example_run_service | C-1 |" in tables
)
assert "| R-1 | Ingress service | runs as | Ingress identity |" in tables
assert "| K-1 | E-1 | The service runs at least two instances. | NFR-1 |" in tables
def test_a_table_cell_cannot_break_its_row():
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
graph["checks"][0]["statement"] = "a | b\nc"
assert "| a \\| b c |" in render_tables(parse_graph(json.dumps(graph)))
async def test_the_engine_writes_the_tables_and_the_diagram_on_submission():
tools = tools_of(K.TARGET_ARCHITECTURE, {})
result = await tools["submit_deliverable"].handler(
content="## Design rationale\nOne service, one identity.\n",
attachments={
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
},
)
assert isinstance(result.outcome, Submitted)
assert result.outcome.content.startswith("## Design rationale")
assert "## Checks" in result.outcome.content
assert result.outcome.attachments["architecture.mmd"].startswith("flowchart")
async def test_the_graph_is_built_a_few_items_at_a_time_and_attached_on_submission():
working: dict = {}
tools = tools_of(K.TARGET_ARCHITECTURE, working)
await tools["graph_put"].handler(elements=GRAPH["elements"])
await tools["graph_put"].handler(relationships=GRAPH["relationships"])
put = await tools["graph_put"].handler(checks=GRAPH["checks"])
assert put.message == (
"Added 2, replaced 0. The graph has 2 Elements, 1 Relationships and 2 Checks."
)
result = await tools["submit_deliverable"].handler(
content="## Design rationale\nReasons.\n"
)
assert isinstance(result.outcome, Submitted)
submitted = json.loads(result.outcome.attachments["target-architecture.json"])
assert [item["id"] for item in submitted["checks"]] == ["K-1", "K-2"]
async def test_a_revision_amends_the_graph_that_was_submitted():
working: dict = {}
tools = tools_of(K.TARGET_ARCHITECTURE, working)
await tools["submit_deliverable"].handler(
content="## Design rationale\nReasons.\n",
attachments={
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
},
)
# A later attempt, with the working memory the Execution kept.
tools = tools_of(K.TARGET_ARCHITECTURE, working)
changed = {**GRAPH["checks"][0], "statement": "min_instance_count is at least 2"}
put = await tools["graph_put"].handler(checks=[changed])
read = await tools["graph_read"].handler()
assert put.message.startswith("Added 0, replaced 1.")
assert "min_instance_count is at least 2" in read.message
assert len(json.loads(read.message)["checks"]) == 2
async def test_graph_items_can_be_removed_and_an_item_without_an_id_is_refused():
working: dict = {}
tools = {tool.name: tool for tool in graph_tools(working, GRAPH_SHAPE)}
await tools["graph_put"].handler(
elements=GRAPH["elements"],
not_applicable=[{"recommendation": "DR-9", "reason": "Single region."}],
)
removed = await tools["graph_remove"].handler(ids=["E-2", "DR-9"])
refused = await tools["graph_put"].handler(checks=[{"statement": "no id"}])
assert removed.message.startswith("Removed 2.")
assert refused.message.startswith("Refused") and "checks[0]" in refused.message
assert [e["id"] for e in working["graph"]["elements"]] == ["E-1"]
# --- files are written once and changed in place ----------------------------------------
async def test_a_file_is_written_once_and_changed_in_place():
working: dict = {}
tools = {tool.name: tool for tool in file_tools(working)}
await tools["write_file"].handler(
path="terraform/main.tf", content="a = 1\nb = 1\n"
)
edited = await tools["edit_file"].handler(
path="terraform/main.tf", old="b = 1", new="b = 2"
)
listing = await tools["read_file"].handler()
assert edited.message == "Edited terraform/main.tf."
assert working["files"] == {"terraform/main.tf": "a = 1\nb = 2\n"}
assert "- terraform/main.tf (2 lines)" in listing.message
async def test_an_edit_that_does_not_match_exactly_once_changes_nothing():
working: dict = {"files": {"main.tf": "x = 1\nx = 1\n"}}
tools = {tool.name: tool for tool in file_tools(working)}
twice = await tools["edit_file"].handler(path="main.tf", old="x = 1", new="x = 2")
never = await tools["edit_file"].handler(path="main.tf", old="y", new="z")
absent = await tools["edit_file"].handler(path="other.tf", old="x", new="y")
unsafe = await tools["write_file"].handler(path="../main.tf", content="x")
assert "it matches 2" in twice.message and "it matches 0" in never.message
assert absent.message.startswith("Refused") and unsafe.message.startswith("Refused")
assert working["files"] == {"main.tf": "x = 1\nx = 1\n"}
async def test_the_working_copy_is_what_is_submitted():
working: dict = {}
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
for path in ("terraform/main.tf", "terraform/variables.tf"):
await tools["write_file"].handler(path=path, content=f"# {path}\n")
await tools["write_file"].handler(
path="implementation.json", content=SAMPLE_DATA["implementation.json"]
)
content = sample_submission(K.DEPLOYMENT_GUIDANCE).args["content"]
result = await tools["submit_deliverable"].handler(content=content)
assert isinstance(result.outcome, Submitted)
assert set(result.outcome.attachments) == {
"terraform/main.tf",
"terraform/variables.tf",
"implementation.json",
}
async def test_a_file_passed_on_submission_replaces_its_working_copy():
working: dict = {"files": {"terraform/main.tf": "old\n"}}
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
call = sample_submission(K.DEPLOYMENT_GUIDANCE)
result = await tools["submit_deliverable"].handler(**call.args)
assert isinstance(result.outcome, Submitted)
assert result.outcome.attachments["terraform/main.tf"] != "old\n"
assert working["files"]["terraform/main.tf"] != "old\n"
def test_only_a_specialist_that_owns_files_can_write_them():
assert "write_file" in tools_of(K.DEPLOYMENT_GUIDANCE, {})
assert "write_file" not in tools_of(K.VALIDATION_RESULTS, {})
assert "graph_put" in tools_of(K.TARGET_ARCHITECTURE, {})
assert "graph_put" not in tools_of(K.DEPLOYMENT_GUIDANCE, {})
# --- Terraform runs on the working copy -------------------------------------------------
class Runner(TerraformRunner):
def __init__(self) -> None:
super().__init__()
self.validated: list[dict[str, str]] = []
async def validate(self, files):
self.validated.append(dict(files))
return ToolResult("terraform validate passed:\n")
async def formatted(self, files):
return {name: content.strip() + "\n" for name, content in files.items()}
async def test_terraform_checks_take_no_files_when_there_is_a_working_copy():
runner = Runner()
files = {"terraform/main.tf": "x = 1\n", "implementation.json": "[]"}
(validate,) = local_tools(["terraform_validate"], runner, files)
await validate.handler()
assert validate.parameters["required"] == []
assert "pass no files" in validate.description
# Only the Terraform files are checked.
assert runner.validated == [{"terraform/main.tf": "x = 1\n"}]
async def test_formatting_changes_the_working_copy_and_returns_only_names():
files = {"terraform/main.tf": " x = 1 \n"}
(fmt,) = local_tools(["terraform_fmt"], Runner(), files)
result = await fmt.handler()
assert files == {"terraform/main.tf": "x = 1\n"}
assert result.message == (
"terraform fmt reformatted 1 file(s) in your working copy: terraform/main.tf."
)
async def test_files_passed_anyway_are_written_to_the_working_copy_first():
runner = Runner()
files: dict[str, str] = {}
(validate,) = local_tools(["terraform_validate"], runner, files)
await validate.handler(files=[{"path": "terraform/main.tf", "content": "y"}])
assert files == {"terraform/main.tf": "y"}
assert runner.validated == [files]
def roles_with(tools_by_role: dict[str, tuple[str, ...]]) -> dict[str, RoleSpec]:
return {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(
spec.specialist,
"Produce.",
budget=15,
tools=tools_by_role.get(spec.specialist, ()),
)
for spec in DEPENDENCY_GRAPH.values()
},
}
async def test_the_validator_checks_the_submitted_code_without_being_sent_it(tmp_path):
runner = Runner()
roles = roles_with({"validator": ("terraform_validate",)})
turns = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
turns["validator"] = [
[Call("terraform_validate", {}), sample_submission(K.VALIDATION_RESULTS)]
]
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
tool_factory=make_tool_factory(
WORKFLOW,
runtime,
roles,
local_tools=lambda working_copy: local_tools(
["terraform_validate"], runner, working_copy
),
),
)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert runner.validated == [
{
"terraform/main.tf": "# terraform/main.tf\n",
"terraform/variables.tf": "# terraform/variables.tf\n",
}
]
# The code is the code author's: validation attaches its verdicts and nothing else.
validation = result.state["deliverables"]["validation_results"]
assert set(validation["attachments"]) == {"verdicts.json"}
# --- each Specialist reads what it works from -------------------------------------------
def submitted() -> dict[str, dict]:
return {
str(kind): {
"content": f"# {kind}\nProse.\n",
"attachments": {
name: SAMPLE_DATA.get(name, f"# {name}\n") for name in spec.attachments
},
}
for kind, spec in DEPENDENCY_GRAPH.items()
}
def test_the_code_is_written_from_the_graph_and_the_ids_alone():
given = dependencies_for(K.DEPLOYMENT_GUIDANCE, submitted())
assert given["target_architecture"] == {
"attachments": {
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
}
}
assert all("content" not in deliverable for deliverable in given.values())
assert set(given["requirements"]["attachments"]) == {"requirements.json"}
def test_a_direct_dependency_is_read_whole_and_the_rest_as_data():
given = dependencies_for(K.TARGET_ARCHITECTURE, submitted())
assert "content" in given["product_selection"]
assert "content" in given["design_recommendations"]
assert "content" not in given["requirements"]
assert "content" not in given["technical_decomposition"]
def test_the_validator_reads_the_code_whole_and_the_graph_as_data():
given = dependencies_for(K.VALIDATION_RESULTS, submitted())
assert "terraform/main.tf" in given["deployment_guidance"]["attachments"]
assert "content" not in given["target_architecture"]
# A Deliverable with no data file has nothing to trim to, so it comes whole.
assert "content" in given["architecture_description"]
def test_the_solution_guide_is_assembled_from_every_deliverable_whole():
given = dependencies_for(K.SOLUTION_GUIDE, submitted())
# Every Deliverable but the guide itself and the plan produced beside it.
assert len(given) == len(DEPENDENCY_GRAPH) - 2
assert all("content" in deliverable for deliverable in given.values())
async def test_a_specialist_is_told_what_its_working_copy_already_holds(tmp_path):
turns = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
failing = sample_submission(K.VALIDATION_RESULTS)
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
verdicts["checks"][0].update(verdict="fail", evidence="min instances is 1")
turns["validator"] = [
[
Call(
failing.tool,
{
**failing.args,
"attachments": {"verdicts.json": json.dumps(verdicts)},
},
)
],
[sample_submission(K.VALIDATION_RESULTS)],
]
turns["iac-author"] = [[sample_submission(K.DEPLOYMENT_GUIDANCE)]] * 2
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=roles_with({}),
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
await engine.start("Ingest webhooks", pre_approved=True)
first, second = (run for run in runtime.runs if run.role == "iac-author")
assert first.task.context["working_copy"] is None
assert second.task.context["working_copy"] == {
"files": ["implementation.json", "terraform/main.tf", "terraform/variables.tf"]
}
async def test_a_deliverable_given_as_data_can_be_read_whole_on_demand():
library = {
"requirements": {
"content": "# requirements\nProse.\n",
"attachments": {"requirements.json": "[]"},
}
}
tools = {
tool.name: tool
for tool in specialist_tools(
WORKFLOW, K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False, library=library
)
}
read = await tools["read_deliverable"].handler(kind="requirements")
refused = await tools["read_deliverable"].handler(kind="solution_guide")
assert (
read.message.startswith("# requirements")
and "requirements.json" in read.message
)
assert refused.message.startswith("Refused")
# With nothing trimmed there is nothing to read back.
assert "read_deliverable" not in tools_of(K.REQUIREMENTS, {})
async def test_the_engine_offers_the_trimmed_deliverables_for_reading(tmp_path):
turns = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=roles_with({}),
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
await engine.start("Ingest webhooks", pre_approved=True)
author = next(run for run in runtime.runs if run.role == "iac-author")
packager = next(run for run in runtime.runs if run.role == "packager")
assert "read_deliverable" in author.tools
# The author is shown the graph alone; the rest it can read when it needs to.
assert list(author.task.context["dependencies"]) == ["target_architecture"]
assert "content" not in author.task.context["dependencies"]["target_architecture"]
assert "read_deliverable" not in packager.tools