"""A Specialist's working copy, and what each Specialist is given to read. Three things made an Execution slow: a Specialist rewrote a whole Deliverable in one reply, sent every file again on each tool call, and read every earlier Deliverable in full. A working copy and trimmed reading remove each of them. """ import json from langgraph.checkpoint.memory import InMemorySaver from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime from execution_engine.adk.runtime import NoDecider, RoleSpec, Submitted, ToolResult from execution_engine.agent import ExecutionEngine from execution_engine.nodes.router import Router, Thresholds from execution_engine.recorder import InMemoryRecorder from execution_engine.tools.execution import specialist_tools from execution_engine.tools.toolkit import make_tool_factory from execution_engine.tools.workspace import file_tools, graph_tools from tests.reference_workflow.hooks import GRAPH_SHAPE from tests.reference_workflow.local import TerraformRunner, local_tools from tests.reference_workflow.target_architecture import parse_graph, render_tables from tests.reference_workflow.workflow import ( DEPENDENCY_GRAPH, SAMPLE_DATA, WORKFLOW, DeliverableKind, dependencies_for, sample_submission, ) K = DeliverableKind GRAPH = json.loads(SAMPLE_DATA["target-architecture.json"]) UPSTREAM = { "requirements": { "attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]} }, "technical_decomposition": { "attachments": {"components.json": SAMPLE_DATA["components.json"]} }, "product_selection": { "attachments": {"products.json": SAMPLE_DATA["products.json"]} }, "design_recommendations": { "attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]} }, "target_architecture": { "attachments": { "target-architecture.json": SAMPLE_DATA["target-architecture.json"] } }, } def tools_of(kind, working, upstream=UPSTREAM, extra=()): return { tool.name: tool for tool in specialist_tools( WORKFLOW, kind, extra, allow_cloud_read=False, upstream=upstream, working=working, ) } # --- the Target architecture is written once ------------------------------------------- def test_tables_are_rendered_from_the_graph(): tables = render_tables(parse_graph(SAMPLE_DATA["target-architecture.json"])) assert "## Elements" in tables and "## Relationships" in tables assert ( "| E-1 | Ingress service | Example Run | example_run_service | C-1 |" in tables ) assert "| R-1 | Ingress service | runs as | Ingress identity |" in tables assert "| K-1 | E-1 | The service runs at least two instances. | NFR-1 |" in tables def test_a_table_cell_cannot_break_its_row(): graph = json.loads(SAMPLE_DATA["target-architecture.json"]) graph["checks"][0]["statement"] = "a | b\nc" assert "| a \\| b c |" in render_tables(parse_graph(json.dumps(graph))) async def test_the_engine_writes_the_tables_and_the_diagram_on_submission(): tools = tools_of(K.TARGET_ARCHITECTURE, {}) result = await tools["submit_deliverable"].handler( content="## Design rationale\nOne service, one identity.\n", attachments={ "target-architecture.json": SAMPLE_DATA["target-architecture.json"] }, ) assert isinstance(result.outcome, Submitted) assert result.outcome.content.startswith("## Design rationale") assert "## Checks" in result.outcome.content assert result.outcome.attachments["architecture.mmd"].startswith("flowchart") async def test_the_graph_is_built_a_few_items_at_a_time_and_attached_on_submission(): working: dict = {} tools = tools_of(K.TARGET_ARCHITECTURE, working) await tools["graph_put"].handler(elements=GRAPH["elements"]) await tools["graph_put"].handler(relationships=GRAPH["relationships"]) put = await tools["graph_put"].handler(checks=GRAPH["checks"]) assert put.message == ( "Added 2, replaced 0. The graph has 2 Elements, 1 Relationships and 2 Checks." ) result = await tools["submit_deliverable"].handler( content="## Design rationale\nReasons.\n" ) assert isinstance(result.outcome, Submitted) submitted = json.loads(result.outcome.attachments["target-architecture.json"]) assert [item["id"] for item in submitted["checks"]] == ["K-1", "K-2"] async def test_a_revision_amends_the_graph_that_was_submitted(): working: dict = {} tools = tools_of(K.TARGET_ARCHITECTURE, working) await tools["submit_deliverable"].handler( content="## Design rationale\nReasons.\n", attachments={ "target-architecture.json": SAMPLE_DATA["target-architecture.json"] }, ) # A later attempt, with the working memory the Execution kept. tools = tools_of(K.TARGET_ARCHITECTURE, working) changed = {**GRAPH["checks"][0], "statement": "min_instance_count is at least 2"} put = await tools["graph_put"].handler(checks=[changed]) read = await tools["graph_read"].handler() assert put.message.startswith("Added 0, replaced 1.") assert "min_instance_count is at least 2" in read.message assert len(json.loads(read.message)["checks"]) == 2 async def test_graph_items_can_be_removed_and_an_item_without_an_id_is_refused(): working: dict = {} tools = {tool.name: tool for tool in graph_tools(working, GRAPH_SHAPE)} await tools["graph_put"].handler( elements=GRAPH["elements"], not_applicable=[{"recommendation": "DR-9", "reason": "Single region."}], ) removed = await tools["graph_remove"].handler(ids=["E-2", "DR-9"]) refused = await tools["graph_put"].handler(checks=[{"statement": "no id"}]) assert removed.message.startswith("Removed 2.") assert refused.message.startswith("Refused") and "checks[0]" in refused.message assert [e["id"] for e in working["graph"]["elements"]] == ["E-1"] # --- files are written once and changed in place ---------------------------------------- async def test_a_file_is_written_once_and_changed_in_place(): working: dict = {} tools = {tool.name: tool for tool in file_tools(working)} await tools["write_file"].handler( path="terraform/main.tf", content="a = 1\nb = 1\n" ) edited = await tools["edit_file"].handler( path="terraform/main.tf", old="b = 1", new="b = 2" ) listing = await tools["read_file"].handler() assert edited.message == "Edited terraform/main.tf." assert working["files"] == {"terraform/main.tf": "a = 1\nb = 2\n"} assert "- terraform/main.tf (2 lines)" in listing.message async def test_an_edit_that_does_not_match_exactly_once_changes_nothing(): working: dict = {"files": {"main.tf": "x = 1\nx = 1\n"}} tools = {tool.name: tool for tool in file_tools(working)} twice = await tools["edit_file"].handler(path="main.tf", old="x = 1", new="x = 2") never = await tools["edit_file"].handler(path="main.tf", old="y", new="z") absent = await tools["edit_file"].handler(path="other.tf", old="x", new="y") unsafe = await tools["write_file"].handler(path="../main.tf", content="x") assert "it matches 2" in twice.message and "it matches 0" in never.message assert absent.message.startswith("Refused") and unsafe.message.startswith("Refused") assert working["files"] == {"main.tf": "x = 1\nx = 1\n"} async def test_the_working_copy_is_what_is_submitted(): working: dict = {} tools = tools_of(K.DEPLOYMENT_GUIDANCE, working) for path in ("terraform/main.tf", "terraform/variables.tf"): await tools["write_file"].handler(path=path, content=f"# {path}\n") await tools["write_file"].handler( path="implementation.json", content=SAMPLE_DATA["implementation.json"] ) content = sample_submission(K.DEPLOYMENT_GUIDANCE).args["content"] result = await tools["submit_deliverable"].handler(content=content) assert isinstance(result.outcome, Submitted) assert set(result.outcome.attachments) == { "terraform/main.tf", "terraform/variables.tf", "implementation.json", } async def test_a_file_passed_on_submission_replaces_its_working_copy(): working: dict = {"files": {"terraform/main.tf": "old\n"}} tools = tools_of(K.DEPLOYMENT_GUIDANCE, working) call = sample_submission(K.DEPLOYMENT_GUIDANCE) result = await tools["submit_deliverable"].handler(**call.args) assert isinstance(result.outcome, Submitted) assert result.outcome.attachments["terraform/main.tf"] != "old\n" assert working["files"]["terraform/main.tf"] != "old\n" def test_only_a_specialist_that_owns_files_can_write_them(): assert "write_file" in tools_of(K.DEPLOYMENT_GUIDANCE, {}) assert "write_file" not in tools_of(K.VALIDATION_RESULTS, {}) assert "graph_put" in tools_of(K.TARGET_ARCHITECTURE, {}) assert "graph_put" not in tools_of(K.DEPLOYMENT_GUIDANCE, {}) # --- Terraform runs on the working copy ------------------------------------------------- class Runner(TerraformRunner): def __init__(self) -> None: super().__init__() self.validated: list[dict[str, str]] = [] async def validate(self, files): self.validated.append(dict(files)) return ToolResult("terraform validate passed:\n") async def formatted(self, files): return {name: content.strip() + "\n" for name, content in files.items()} async def test_terraform_checks_take_no_files_when_there_is_a_working_copy(): runner = Runner() files = {"terraform/main.tf": "x = 1\n", "implementation.json": "[]"} (validate,) = local_tools(["terraform_validate"], runner, files) await validate.handler() assert validate.parameters["required"] == [] assert "pass no files" in validate.description # Only the Terraform files are checked. assert runner.validated == [{"terraform/main.tf": "x = 1\n"}] async def test_formatting_changes_the_working_copy_and_returns_only_names(): files = {"terraform/main.tf": " x = 1 \n"} (fmt,) = local_tools(["terraform_fmt"], Runner(), files) result = await fmt.handler() assert files == {"terraform/main.tf": "x = 1\n"} assert result.message == ( "terraform fmt reformatted 1 file(s) in your working copy: terraform/main.tf." ) async def test_files_passed_anyway_are_written_to_the_working_copy_first(): runner = Runner() files: dict[str, str] = {} (validate,) = local_tools(["terraform_validate"], runner, files) await validate.handler(files=[{"path": "terraform/main.tf", "content": "y"}]) assert files == {"terraform/main.tf": "y"} assert runner.validated == [files] def roles_with(tools_by_role: dict[str, tuple[str, ...]]) -> dict[str, RoleSpec]: return { "orchestrator": RoleSpec("orchestrator", "Unused.", budget=40), **{ spec.specialist: RoleSpec( spec.specialist, "Produce.", budget=15, tools=tools_by_role.get(spec.specialist, ()), ) for spec in DEPENDENCY_GRAPH.values() }, } async def test_the_validator_checks_the_submitted_code_without_being_sent_it(tmp_path): runner = Runner() roles = roles_with({"validator": ("terraform_validate",)}) turns = { spec.specialist: [[sample_submission(kind)]] for kind, spec in DEPENDENCY_GRAPH.items() } turns["validator"] = [ [Call("terraform_validate", {}), sample_submission(K.VALIDATION_RESULTS)] ] runtime = FakeAgentRuntime(turns) engine = ExecutionEngine( workflow=WORKFLOW, runtime=runtime, roles=roles, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), tool_factory=make_tool_factory( WORKFLOW, runtime, roles, local_tools=lambda working_copy: local_tools( ["terraform_validate"], runner, working_copy ), ), ) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" assert runner.validated == [ { "terraform/main.tf": "# terraform/main.tf\n", "terraform/variables.tf": "# terraform/variables.tf\n", } ] # The code is the code author's: validation attaches its verdicts and nothing else. validation = result.state["deliverables"]["validation_results"] assert set(validation["attachments"]) == {"verdicts.json"} # --- each Specialist reads what it works from ------------------------------------------- def submitted() -> dict[str, dict]: return { str(kind): { "content": f"# {kind}\nProse.\n", "attachments": { name: SAMPLE_DATA.get(name, f"# {name}\n") for name in spec.attachments }, } for kind, spec in DEPENDENCY_GRAPH.items() } def test_the_code_is_written_from_the_graph_and_the_ids_alone(): given = dependencies_for(K.DEPLOYMENT_GUIDANCE, submitted()) assert given["target_architecture"] == { "attachments": { "target-architecture.json": SAMPLE_DATA["target-architecture.json"] } } assert all("content" not in deliverable for deliverable in given.values()) assert set(given["requirements"]["attachments"]) == {"requirements.json"} def test_a_direct_dependency_is_read_whole_and_the_rest_as_data(): given = dependencies_for(K.TARGET_ARCHITECTURE, submitted()) assert "content" in given["product_selection"] assert "content" in given["design_recommendations"] assert "content" not in given["requirements"] assert "content" not in given["technical_decomposition"] def test_the_validator_reads_the_code_whole_and_the_graph_as_data(): given = dependencies_for(K.VALIDATION_RESULTS, submitted()) assert "terraform/main.tf" in given["deployment_guidance"]["attachments"] assert "content" not in given["target_architecture"] # A Deliverable with no data file has nothing to trim to, so it comes whole. assert "content" in given["architecture_description"] def test_the_solution_guide_is_assembled_from_every_deliverable_whole(): given = dependencies_for(K.SOLUTION_GUIDE, submitted()) # Every Deliverable but the guide itself and the plan produced beside it. assert len(given) == len(DEPENDENCY_GRAPH) - 2 assert all("content" in deliverable for deliverable in given.values()) async def test_a_specialist_is_told_what_its_working_copy_already_holds(tmp_path): turns = { spec.specialist: [[sample_submission(kind)]] for kind, spec in DEPENDENCY_GRAPH.items() } failing = sample_submission(K.VALIDATION_RESULTS) verdicts = json.loads(SAMPLE_DATA["verdicts.json"]) verdicts["checks"][0].update(verdict="fail", evidence="min instances is 1") turns["validator"] = [ [ Call( failing.tool, { **failing.args, "attachments": {"verdicts.json": json.dumps(verdicts)}, }, ) ], [sample_submission(K.VALIDATION_RESULTS)], ] turns["iac-author"] = [[sample_submission(K.DEPLOYMENT_GUIDANCE)]] * 2 runtime = FakeAgentRuntime(turns) engine = ExecutionEngine( workflow=WORKFLOW, runtime=runtime, roles=roles_with({}), recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), ) await engine.start("Ingest webhooks", pre_approved=True) first, second = (run for run in runtime.runs if run.role == "iac-author") assert first.task.context["working_copy"] is None assert second.task.context["working_copy"] == { "files": ["implementation.json", "terraform/main.tf", "terraform/variables.tf"] } async def test_a_deliverable_given_as_data_can_be_read_whole_on_demand(): library = { "requirements": { "content": "# requirements\nProse.\n", "attachments": {"requirements.json": "[]"}, } } tools = { tool.name: tool for tool in specialist_tools( WORKFLOW, K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False, library=library ) } read = await tools["read_deliverable"].handler(kind="requirements") refused = await tools["read_deliverable"].handler(kind="solution_guide") assert ( read.message.startswith("# requirements") and "requirements.json" in read.message ) assert refused.message.startswith("Refused") # With nothing trimmed there is nothing to read back. assert "read_deliverable" not in tools_of(K.REQUIREMENTS, {}) async def test_the_engine_offers_the_trimmed_deliverables_for_reading(tmp_path): turns = { spec.specialist: [[sample_submission(kind)]] for kind, spec in DEPENDENCY_GRAPH.items() } runtime = FakeAgentRuntime(turns) engine = ExecutionEngine( workflow=WORKFLOW, runtime=runtime, roles=roles_with({}), recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), ) await engine.start("Ingest webhooks", pre_approved=True) author = next(run for run in runtime.runs if run.role == "iac-author") packager = next(run for run in runtime.runs if run.role == "packager") assert "read_deliverable" in author.tools # The author is shown the graph alone; the rest it can read when it needs to. assert list(author.task.context["dependencies"]) == ["target_architecture"] assert "content" not in author.task.context["dependencies"]["target_architecture"] assert "read_deliverable" not in packager.tools