481 lines
17 KiB
Python
481 lines
17 KiB
Python
"""A Specialist's working copy, and what each Specialist is given to read.
|
|
|
|
Three things made an Execution slow: a Specialist rewrote a whole Deliverable in one
|
|
reply, sent every file again on each tool call, and read every earlier Deliverable in
|
|
full. A working copy and trimmed reading remove each of them.
|
|
"""
|
|
|
|
import json
|
|
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from app.adk.fake_runtime import (
|
|
SAMPLE_DATA,
|
|
Call,
|
|
FakeAgentRuntime,
|
|
sample_submission,
|
|
)
|
|
from app.adk.runtime import NoDecider, RoleSpec, Submitted, ToolResult
|
|
from app.agent import ExecutionEngine
|
|
from app.nodes.router import Router, Thresholds
|
|
from app.recorder import InMemoryRecorder
|
|
from app.tools.execution import specialist_tools
|
|
from app.tools.local import TerraformRunner, local_tools
|
|
from app.tools.toolkit import make_tool_factory
|
|
from app.tools.workspace import file_tools, graph_tools
|
|
from app.workflows.deliverables import (
|
|
DEPENDENCY_GRAPH,
|
|
DeliverableKind,
|
|
dependencies_for,
|
|
)
|
|
from app.workflows.target_architecture import parse_graph, render_tables
|
|
|
|
K = DeliverableKind
|
|
GRAPH = json.loads(SAMPLE_DATA["target-architecture.json"])
|
|
UPSTREAM = {
|
|
"requirements": {
|
|
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
|
},
|
|
"technical_decomposition": {
|
|
"attachments": {"components.json": SAMPLE_DATA["components.json"]}
|
|
},
|
|
"product_selection": {
|
|
"attachments": {"products.json": SAMPLE_DATA["products.json"]}
|
|
},
|
|
"design_recommendations": {
|
|
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
|
|
},
|
|
"target_architecture": {
|
|
"attachments": {
|
|
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
def tools_of(kind, working, upstream=UPSTREAM, extra=()):
|
|
return {
|
|
tool.name: tool
|
|
for tool in specialist_tools(
|
|
kind, extra, allow_cloud_read=False, upstream=upstream, working=working
|
|
)
|
|
}
|
|
|
|
|
|
# --- the Target architecture is written once -------------------------------------------
|
|
|
|
|
|
def test_tables_are_rendered_from_the_graph():
|
|
tables = render_tables(parse_graph(SAMPLE_DATA["target-architecture.json"]))
|
|
|
|
assert "## Elements" in tables and "## Relationships" in tables
|
|
assert (
|
|
"| E-1 | Ingress service | Example Run | example_run_service | C-1 |" in tables
|
|
)
|
|
assert "| R-1 | Ingress service | runs as | Ingress identity |" in tables
|
|
assert "| K-1 | E-1 | The service runs at least two instances. | NFR-1 |" in tables
|
|
|
|
|
|
def test_a_table_cell_cannot_break_its_row():
|
|
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
|
graph["checks"][0]["statement"] = "a | b\nc"
|
|
|
|
assert "| a \\| b c |" in render_tables(parse_graph(json.dumps(graph)))
|
|
|
|
|
|
async def test_the_engine_writes_the_tables_and_the_diagram_on_submission():
|
|
tools = tools_of(K.TARGET_ARCHITECTURE, {})
|
|
|
|
result = await tools["submit_deliverable"].handler(
|
|
content="## Design rationale\nOne service, one identity.\n",
|
|
attachments={
|
|
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
|
},
|
|
)
|
|
|
|
assert isinstance(result.outcome, Submitted)
|
|
assert result.outcome.content.startswith("## Design rationale")
|
|
assert "## Checks" in result.outcome.content
|
|
assert result.outcome.attachments["architecture.mmd"].startswith("flowchart")
|
|
|
|
|
|
async def test_the_graph_is_built_a_few_items_at_a_time_and_attached_on_submission():
|
|
working: dict = {}
|
|
tools = tools_of(K.TARGET_ARCHITECTURE, working)
|
|
|
|
await tools["graph_put"].handler(elements=GRAPH["elements"])
|
|
await tools["graph_put"].handler(relationships=GRAPH["relationships"])
|
|
put = await tools["graph_put"].handler(checks=GRAPH["checks"])
|
|
assert put.message == (
|
|
"Added 2, replaced 0. The graph has 2 Elements, 1 Relationships and 2 Checks."
|
|
)
|
|
|
|
result = await tools["submit_deliverable"].handler(
|
|
content="## Design rationale\nReasons.\n"
|
|
)
|
|
|
|
assert isinstance(result.outcome, Submitted)
|
|
submitted = json.loads(result.outcome.attachments["target-architecture.json"])
|
|
assert [item["id"] for item in submitted["checks"]] == ["K-1", "K-2"]
|
|
|
|
|
|
async def test_a_revision_amends_the_graph_that_was_submitted():
|
|
working: dict = {}
|
|
tools = tools_of(K.TARGET_ARCHITECTURE, working)
|
|
await tools["submit_deliverable"].handler(
|
|
content="## Design rationale\nReasons.\n",
|
|
attachments={
|
|
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
|
},
|
|
)
|
|
|
|
# A later attempt, with the working memory the Execution kept.
|
|
tools = tools_of(K.TARGET_ARCHITECTURE, working)
|
|
changed = {**GRAPH["checks"][0], "statement": "min_instance_count is at least 2"}
|
|
put = await tools["graph_put"].handler(checks=[changed])
|
|
read = await tools["graph_read"].handler()
|
|
|
|
assert put.message.startswith("Added 0, replaced 1.")
|
|
assert "min_instance_count is at least 2" in read.message
|
|
assert len(json.loads(read.message)["checks"]) == 2
|
|
|
|
|
|
async def test_graph_items_can_be_removed_and_an_item_without_an_id_is_refused():
|
|
working: dict = {}
|
|
tools = {tool.name: tool for tool in graph_tools(working)}
|
|
await tools["graph_put"].handler(
|
|
elements=GRAPH["elements"],
|
|
not_applicable=[{"recommendation": "DR-9", "reason": "Single region."}],
|
|
)
|
|
|
|
removed = await tools["graph_remove"].handler(ids=["E-2", "DR-9"])
|
|
refused = await tools["graph_put"].handler(checks=[{"statement": "no id"}])
|
|
|
|
assert removed.message.startswith("Removed 2.")
|
|
assert refused.message.startswith("Refused") and "checks[0]" in refused.message
|
|
assert [e["id"] for e in working["graph"]["elements"]] == ["E-1"]
|
|
|
|
|
|
# --- files are written once and changed in place ----------------------------------------
|
|
|
|
|
|
async def test_a_file_is_written_once_and_changed_in_place():
|
|
working: dict = {}
|
|
tools = {tool.name: tool for tool in file_tools(working)}
|
|
|
|
await tools["write_file"].handler(
|
|
path="terraform/main.tf", content="a = 1\nb = 1\n"
|
|
)
|
|
edited = await tools["edit_file"].handler(
|
|
path="terraform/main.tf", old="b = 1", new="b = 2"
|
|
)
|
|
listing = await tools["read_file"].handler()
|
|
|
|
assert edited.message == "Edited terraform/main.tf."
|
|
assert working["files"] == {"terraform/main.tf": "a = 1\nb = 2\n"}
|
|
assert "- terraform/main.tf (2 lines)" in listing.message
|
|
|
|
|
|
async def test_an_edit_that_does_not_match_exactly_once_changes_nothing():
|
|
working: dict = {"files": {"main.tf": "x = 1\nx = 1\n"}}
|
|
tools = {tool.name: tool for tool in file_tools(working)}
|
|
|
|
twice = await tools["edit_file"].handler(path="main.tf", old="x = 1", new="x = 2")
|
|
never = await tools["edit_file"].handler(path="main.tf", old="y", new="z")
|
|
absent = await tools["edit_file"].handler(path="other.tf", old="x", new="y")
|
|
unsafe = await tools["write_file"].handler(path="../main.tf", content="x")
|
|
|
|
assert "it matches 2" in twice.message and "it matches 0" in never.message
|
|
assert absent.message.startswith("Refused") and unsafe.message.startswith("Refused")
|
|
assert working["files"] == {"main.tf": "x = 1\nx = 1\n"}
|
|
|
|
|
|
async def test_the_working_copy_is_what_is_submitted():
|
|
working: dict = {}
|
|
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
|
|
for path in ("terraform/main.tf", "terraform/variables.tf"):
|
|
await tools["write_file"].handler(path=path, content=f"# {path}\n")
|
|
await tools["write_file"].handler(
|
|
path="implementation.json", content=SAMPLE_DATA["implementation.json"]
|
|
)
|
|
content = sample_submission(K.DEPLOYMENT_GUIDANCE).args["content"]
|
|
|
|
result = await tools["submit_deliverable"].handler(content=content)
|
|
|
|
assert isinstance(result.outcome, Submitted)
|
|
assert set(result.outcome.attachments) == {
|
|
"terraform/main.tf",
|
|
"terraform/variables.tf",
|
|
"implementation.json",
|
|
}
|
|
|
|
|
|
async def test_a_file_passed_on_submission_replaces_its_working_copy():
|
|
working: dict = {"files": {"terraform/main.tf": "old\n"}}
|
|
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
|
|
call = sample_submission(K.DEPLOYMENT_GUIDANCE)
|
|
|
|
result = await tools["submit_deliverable"].handler(**call.args)
|
|
|
|
assert isinstance(result.outcome, Submitted)
|
|
assert result.outcome.attachments["terraform/main.tf"] != "old\n"
|
|
assert working["files"]["terraform/main.tf"] != "old\n"
|
|
|
|
|
|
def test_only_a_specialist_that_owns_files_can_write_them():
|
|
assert "write_file" in tools_of(K.DEPLOYMENT_GUIDANCE, {})
|
|
assert "write_file" not in tools_of(K.VALIDATION_RESULTS, {})
|
|
assert "graph_put" in tools_of(K.TARGET_ARCHITECTURE, {})
|
|
assert "graph_put" not in tools_of(K.DEPLOYMENT_GUIDANCE, {})
|
|
|
|
|
|
# --- Terraform runs on the working copy -------------------------------------------------
|
|
|
|
|
|
class Runner(TerraformRunner):
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
self.validated: list[dict[str, str]] = []
|
|
|
|
async def validate(self, files):
|
|
self.validated.append(dict(files))
|
|
return ToolResult("terraform validate passed:\n")
|
|
|
|
async def formatted(self, files):
|
|
return {name: content.strip() + "\n" for name, content in files.items()}
|
|
|
|
|
|
async def test_terraform_checks_take_no_files_when_there_is_a_working_copy():
|
|
runner = Runner()
|
|
files = {"terraform/main.tf": "x = 1\n", "implementation.json": "[]"}
|
|
(validate,) = local_tools(["terraform_validate"], runner, files)
|
|
|
|
await validate.handler()
|
|
|
|
assert validate.parameters["required"] == []
|
|
assert "pass no files" in validate.description
|
|
# Only the Terraform files are checked.
|
|
assert runner.validated == [{"terraform/main.tf": "x = 1\n"}]
|
|
|
|
|
|
async def test_formatting_changes_the_working_copy_and_returns_only_names():
|
|
files = {"terraform/main.tf": " x = 1 \n"}
|
|
(fmt,) = local_tools(["terraform_fmt"], Runner(), files)
|
|
|
|
result = await fmt.handler()
|
|
|
|
assert files == {"terraform/main.tf": "x = 1\n"}
|
|
assert result.message == (
|
|
"terraform fmt reformatted 1 file(s) in your working copy: terraform/main.tf."
|
|
)
|
|
|
|
|
|
async def test_files_passed_anyway_are_written_to_the_working_copy_first():
|
|
runner = Runner()
|
|
files: dict[str, str] = {}
|
|
(validate,) = local_tools(["terraform_validate"], runner, files)
|
|
|
|
await validate.handler(files=[{"path": "terraform/main.tf", "content": "y"}])
|
|
|
|
assert files == {"terraform/main.tf": "y"}
|
|
assert runner.validated == [files]
|
|
|
|
|
|
def roles_with(tools_by_role: dict[str, tuple[str, ...]]) -> dict[str, RoleSpec]:
|
|
return {
|
|
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
|
**{
|
|
spec.specialist: RoleSpec(
|
|
spec.specialist,
|
|
"Produce.",
|
|
budget=15,
|
|
tools=tools_by_role.get(spec.specialist, ()),
|
|
)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
|
|
|
|
async def test_the_validator_checks_the_submitted_code_without_being_sent_it(tmp_path):
|
|
runner = Runner()
|
|
roles = roles_with({"validator": ("terraform_validate",)})
|
|
turns = {
|
|
spec.specialist: [[sample_submission(kind)]]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
turns["validator"] = [
|
|
[Call("terraform_validate", {}), sample_submission(K.VALIDATION_RESULTS)]
|
|
]
|
|
runtime = FakeAgentRuntime(turns)
|
|
engine = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=roles,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
tool_factory=make_tool_factory(
|
|
runtime,
|
|
roles,
|
|
{},
|
|
local_tools=lambda working_copy: local_tools(
|
|
["terraform_validate"], runner, working_copy
|
|
),
|
|
),
|
|
)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
assert runner.validated == [
|
|
{
|
|
"terraform/main.tf": "# terraform/main.tf\n",
|
|
"terraform/variables.tf": "# terraform/variables.tf\n",
|
|
}
|
|
]
|
|
# The code is the code author's: validation attaches its verdicts and nothing else.
|
|
validation = result.state["deliverables"]["validation_results"]
|
|
assert set(validation["attachments"]) == {"verdicts.json"}
|
|
|
|
|
|
# --- each Specialist reads what it works from -------------------------------------------
|
|
|
|
|
|
def submitted() -> dict[str, dict]:
|
|
return {
|
|
kind.value: {
|
|
"content": f"# {kind}\nProse.\n",
|
|
"attachments": {
|
|
name: SAMPLE_DATA.get(name, f"# {name}\n") for name in spec.attachments
|
|
},
|
|
}
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
|
|
|
|
def test_the_code_is_written_from_the_graph_and_the_ids_alone():
|
|
given = dependencies_for(K.DEPLOYMENT_GUIDANCE, submitted())
|
|
|
|
assert given["target_architecture"] == {
|
|
"attachments": {
|
|
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
|
}
|
|
}
|
|
assert all("content" not in deliverable for deliverable in given.values())
|
|
assert set(given["requirements"]["attachments"]) == {"requirements.json"}
|
|
|
|
|
|
def test_a_direct_dependency_is_read_whole_and_the_rest_as_data():
|
|
given = dependencies_for(K.TARGET_ARCHITECTURE, submitted())
|
|
|
|
assert "content" in given["product_selection"]
|
|
assert "content" in given["design_recommendations"]
|
|
assert "content" not in given["requirements"]
|
|
assert "content" not in given["technical_decomposition"]
|
|
|
|
|
|
def test_the_validator_reads_the_code_whole_and_the_graph_as_data():
|
|
given = dependencies_for(K.VALIDATION_RESULTS, submitted())
|
|
|
|
assert "terraform/main.tf" in given["deployment_guidance"]["attachments"]
|
|
assert "content" not in given["target_architecture"]
|
|
# A Deliverable with no data file has nothing to trim to, so it comes whole.
|
|
assert "content" in given["architecture_description"]
|
|
|
|
|
|
def test_the_solution_guide_is_assembled_from_every_deliverable_whole():
|
|
given = dependencies_for(K.SOLUTION_GUIDE, submitted())
|
|
|
|
assert len(given) == len(DEPENDENCY_GRAPH) - 1
|
|
assert all("content" in deliverable for deliverable in given.values())
|
|
|
|
|
|
async def test_a_specialist_is_told_what_its_working_copy_already_holds(tmp_path):
|
|
turns = {
|
|
spec.specialist: [[sample_submission(kind)]]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
failing = sample_submission(K.VALIDATION_RESULTS)
|
|
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
|
verdicts["checks"][0].update(verdict="fail", evidence="min instances is 1")
|
|
turns["validator"] = [
|
|
[
|
|
Call(
|
|
failing.tool,
|
|
{
|
|
**failing.args,
|
|
"attachments": {"verdicts.json": json.dumps(verdicts)},
|
|
},
|
|
)
|
|
],
|
|
[sample_submission(K.VALIDATION_RESULTS)],
|
|
]
|
|
turns["iac-author"] = [[sample_submission(K.DEPLOYMENT_GUIDANCE)]] * 2
|
|
runtime = FakeAgentRuntime(turns)
|
|
engine = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=roles_with({}),
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
)
|
|
|
|
await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
first, second = (run for run in runtime.runs if run.role == "iac-author")
|
|
assert first.task.context["working_copy"] is None
|
|
assert second.task.context["working_copy"] == {
|
|
"files": ["implementation.json", "terraform/main.tf", "terraform/variables.tf"]
|
|
}
|
|
|
|
|
|
async def test_a_deliverable_given_as_data_can_be_read_whole_on_demand():
|
|
library = {
|
|
"requirements": {
|
|
"content": "# requirements\nProse.\n",
|
|
"attachments": {"requirements.json": "[]"},
|
|
}
|
|
}
|
|
tools = {
|
|
tool.name: tool
|
|
for tool in specialist_tools(
|
|
K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False, library=library
|
|
)
|
|
}
|
|
|
|
read = await tools["read_deliverable"].handler(kind="requirements")
|
|
refused = await tools["read_deliverable"].handler(kind="solution_guide")
|
|
|
|
assert (
|
|
read.message.startswith("# requirements")
|
|
and "requirements.json" in read.message
|
|
)
|
|
assert refused.message.startswith("Refused")
|
|
# With nothing trimmed there is nothing to read back.
|
|
assert "read_deliverable" not in tools_of(K.REQUIREMENTS, {})
|
|
|
|
|
|
async def test_the_engine_offers_the_trimmed_deliverables_for_reading(tmp_path):
|
|
turns = {
|
|
spec.specialist: [[sample_submission(kind)]]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
runtime = FakeAgentRuntime(turns)
|
|
engine = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=roles_with({}),
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
)
|
|
|
|
await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
author = next(run for run in runtime.runs if run.role == "iac-author")
|
|
packager = next(run for run in runtime.runs if run.role == "packager")
|
|
assert "read_deliverable" in author.tools
|
|
assert "content" not in author.task.context["dependencies"]["requirements"]
|
|
assert "read_deliverable" not in packager.tools
|