This commit is contained in:
@@ -15,7 +15,7 @@ from app.agent import ExecutionEngine
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.tools.local import mermaid_check
|
||||
from app.workflows.chain import validation_outcome
|
||||
from app.workflows.chain import unimplementable_checks, validation_outcome
|
||||
from app.workflows.deliverables import (
|
||||
DEPENDENCY_GRAPH,
|
||||
DeliverableKind,
|
||||
@@ -327,6 +327,54 @@ def test_deployment_guidance_must_account_for_every_check():
|
||||
)
|
||||
|
||||
|
||||
def test_a_check_that_cannot_be_implemented_is_accounted_for_with_its_reason():
|
||||
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
|
||||
implementation = json.dumps(
|
||||
[
|
||||
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
|
||||
{
|
||||
"check": "K-2",
|
||||
"unimplementable": "example_service_account has no such argument",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
missing = check_submission(
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
args["content"],
|
||||
{**args["attachments"], "implementation.json": implementation},
|
||||
upstream(*EVERYTHING_BEFORE_CODE),
|
||||
)
|
||||
|
||||
assert missing == ()
|
||||
assert unimplementable_checks(implementation) == [
|
||||
{"check": "K-2", "reason": "example_service_account has no such argument"}
|
||||
]
|
||||
assert unimplementable_checks(SAMPLE_DATA["implementation.json"]) == []
|
||||
|
||||
|
||||
def test_an_unimplementable_entry_needs_its_reason():
|
||||
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
|
||||
implementation = json.dumps(
|
||||
[
|
||||
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
|
||||
{"check": "K-2", "unimplementable": ""},
|
||||
]
|
||||
)
|
||||
|
||||
missing = check_submission(
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
args["content"],
|
||||
{**args["attachments"], "implementation.json": implementation},
|
||||
upstream(*EVERYTHING_BEFORE_CODE),
|
||||
)
|
||||
|
||||
assert missing == (
|
||||
"implementation.json[0]: needs unimplementable",
|
||||
"implementation.json: Check K-2 is not accounted for",
|
||||
)
|
||||
|
||||
|
||||
def verdicts(**changes) -> str:
|
||||
data = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
data.update(changes)
|
||||
|
||||
598
tests/execution/test_router.py
Normal file
598
tests/execution/test_router.py
Normal file
@@ -0,0 +1,598 @@
|
||||
"""The Orchestrator as a router (ADR-0007): readings, a disposition, then a playbook.
|
||||
|
||||
No language model chooses an action here. Specialists are scripted; the Orchestrator
|
||||
is the Router, with a scripted decider standing in for Jev, or none at all.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import (
|
||||
SAMPLE_DATA,
|
||||
Call,
|
||||
FakeAgentRuntime,
|
||||
FakeDecider,
|
||||
sample_submission,
|
||||
)
|
||||
from app.adk.runtime import Answer, NoDecider, RoleSpec
|
||||
from app.agent import ExecutionEngine
|
||||
from app.nodes.router import (
|
||||
REPLY_WRITER,
|
||||
Router,
|
||||
RuntimeReplyWriter,
|
||||
Thresholds,
|
||||
)
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
||||
|
||||
K = DeliverableKind
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused by the router.", budget=40),
|
||||
REPLY_WRITER: RoleSpec(REPLY_WRITER, "Write the reply.", budget=3),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def specialist(kind: DeliverableKind) -> str:
|
||||
return DEPENDENCY_GRAPH[kind].specialist
|
||||
|
||||
|
||||
def once() -> dict:
|
||||
return {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
|
||||
|
||||
def validation(**changes) -> Call:
|
||||
"""A validation submission whose verdicts differ from the all-pass sample."""
|
||||
call = sample_submission(K.VALIDATION_RESULTS)
|
||||
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
verdicts.update(changes)
|
||||
return Call(
|
||||
call.tool,
|
||||
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
|
||||
)
|
||||
|
||||
|
||||
def failing(check: str = "K-1") -> Call:
|
||||
checks = json.loads(SAMPLE_DATA["verdicts.json"])["checks"]
|
||||
for item in checks:
|
||||
if item["check"] == check:
|
||||
item.update(verdict="fail", evidence="min instances is 1")
|
||||
return validation(checks=checks)
|
||||
|
||||
|
||||
def addition(reason: str) -> Call:
|
||||
return validation(
|
||||
proposed_additions=[
|
||||
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": reason}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def engine_for(tmp_path, turns: dict, decider=None, writer: bool = False, **options):
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
router = Router(
|
||||
decider or NoDecider(),
|
||||
RuntimeReplyWriter(runtime, ROLES[REPLY_WRITER]) if writer else None,
|
||||
Thresholds(),
|
||||
)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=router,
|
||||
**options,
|
||||
)
|
||||
return engine, runtime
|
||||
|
||||
|
||||
def actions(result) -> list[str]:
|
||||
return [decision["action"] for decision in result.state["decisions"]]
|
||||
|
||||
|
||||
def reopened(result) -> list[tuple[str, str]]:
|
||||
return [
|
||||
(event["subject"], event["detail"])
|
||||
for event in result.state["ledger"]["events"]
|
||||
if event["type"] == "reopened"
|
||||
]
|
||||
|
||||
|
||||
def approve(phase: str) -> dict:
|
||||
return {"approval": {"phase": phase, "decision": "approve"}}
|
||||
|
||||
|
||||
# --- the playbook on rules alone -------------------------------------------------------
|
||||
|
||||
|
||||
async def test_an_execution_completes_with_no_model_choosing_an_action(tmp_path):
|
||||
engine, runtime = engine_for(tmp_path, once())
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert "orchestrator" not in {run.role for run in runtime.runs}
|
||||
assert actions(result) == [
|
||||
"delegate", # requirements
|
||||
"delegate", # technical decomposition
|
||||
"request_approval",
|
||||
"delegate", # product selection
|
||||
"delegate", # design recommendations
|
||||
"delegate", # Target architecture
|
||||
"request_approval",
|
||||
"delegate", # description and deployment guidance, together
|
||||
"request_approval",
|
||||
"delegate", # validation
|
||||
"proceed",
|
||||
"request_approval",
|
||||
"delegate", # solution guide
|
||||
"request_approval",
|
||||
"finish",
|
||||
]
|
||||
groups = {decision["group"] for decision in result.state["decisions"]}
|
||||
assert groups == {
|
||||
"work is ready",
|
||||
"a Phase is complete",
|
||||
"validation decided",
|
||||
"packaging is approved",
|
||||
}
|
||||
|
||||
|
||||
async def test_every_decision_is_recorded_with_its_turn_group_and_action(tmp_path):
|
||||
engine, _ = engine_for(tmp_path, once())
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
first = result.state["decisions"][0]
|
||||
assert first == {
|
||||
"turn": 1,
|
||||
"group": "work is ready",
|
||||
"action": "delegate",
|
||||
"judged": {},
|
||||
}
|
||||
assert [d["turn"] for d in result.state["decisions"]] == sorted(
|
||||
d["turn"] for d in result.state["decisions"]
|
||||
)
|
||||
|
||||
|
||||
async def test_when_no_group_applies_a_pre_approved_execution_ends_with_warnings(
|
||||
tmp_path,
|
||||
):
|
||||
turns = {specialist(K.REQUIREMENTS): [[]]} # stops without submitting
|
||||
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
# The Specialist's question is relayed, answered automatically, and it stops
|
||||
# again; the router never invents work.
|
||||
assert result.status == "completed_with_warnings"
|
||||
|
||||
|
||||
# --- Approvals: the summary is written from data, and only data approves ----------------
|
||||
|
||||
|
||||
async def test_the_approval_summary_is_assembled_from_the_data(tmp_path):
|
||||
engine, _ = engine_for(tmp_path, once())
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
design = await engine.resume(first.execution_id, approve("requirements_discovery"))
|
||||
|
||||
assert first.prompt["approval"]["summary"] == (
|
||||
"The requirements_discovery Phase is ready: requirements, "
|
||||
"technical_decomposition submitted. 2 requirements."
|
||||
)
|
||||
summary = design.prompt["approval"]["summary"]
|
||||
assert "Target architecture: 2 Elements, 1 Relationships, 2 Checks." in summary
|
||||
assert design.prompt["approval"]["target_architecture"]["checks"] == 2
|
||||
|
||||
|
||||
async def test_a_model_writes_the_summary_but_decides_nothing(tmp_path):
|
||||
turns = {
|
||||
**once(),
|
||||
REPLY_WRITER: [[Call("write_reply", {"text": "Two requirements, ready."})]],
|
||||
}
|
||||
engine, runtime = engine_for(tmp_path, turns, writer=True)
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
|
||||
assert first.prompt["approval"]["summary"] == "Two requirements, ready."
|
||||
writer_run = next(run for run in runtime.runs if run.role == REPLY_WRITER)
|
||||
assert writer_run.tools == ("write_reply",)
|
||||
assert writer_run.task.context["facts"]["deliverables"] == [
|
||||
"requirements",
|
||||
"technical_decomposition",
|
||||
]
|
||||
|
||||
|
||||
async def test_a_writer_that_fails_does_not_stop_the_approval(tmp_path):
|
||||
engine, _ = engine_for(tmp_path, once(), writer=True) # no writer turn scripted
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
|
||||
assert first.prompt["approval"]["summary"].startswith(
|
||||
"The requirements_discovery Phase is ready"
|
||||
)
|
||||
|
||||
|
||||
async def test_a_text_reply_never_approves_and_the_approval_is_asked_again(tmp_path):
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"asks_change": Answer(0.03, 0.94),
|
||||
"change_target": Answer("requirements", 0.5),
|
||||
"change_clarity": Answer(0.0, 0.9),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, once(), decider)
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
again = await engine.resume(first.execution_id, {"reply": "Looks good to me."})
|
||||
|
||||
assert again.prompt["approval"]["phase"] == "requirements_discovery"
|
||||
assert again.state["ledger"]["phases"]["requirements_discovery"] != "approved"
|
||||
|
||||
|
||||
# --- validation: the code, the graph, or proceed ----------------------------------------
|
||||
|
||||
|
||||
async def test_a_failed_check_sends_the_code_back_and_leaves_the_design_approved(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once()
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [
|
||||
[failing("K-1")],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
engine, runtime = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert reopened(result) == [
|
||||
("deployment_guidance", "K-1 failed: min instances is 1")
|
||||
]
|
||||
assert "revise_code" in actions(result)
|
||||
architect_runs = [
|
||||
run for run in runtime.runs if run.role == specialist(K.TARGET_ARCHITECTURE)
|
||||
]
|
||||
assert len(architect_runs) == 1
|
||||
|
||||
|
||||
async def test_a_check_the_code_cannot_meet_goes_back_to_the_graph_unvalidated(
|
||||
tmp_path,
|
||||
):
|
||||
"""The code's author found the Check names a setting the resource lacks. No
|
||||
validation is spent on it: the Target architecture is revised first."""
|
||||
cannot = sample_submission(K.DEPLOYMENT_GUIDANCE)
|
||||
implementation = json.loads(SAMPLE_DATA["implementation.json"])
|
||||
implementation[1] = {"check": "K-2", "unimplementable": "no such argument"}
|
||||
turns = once()
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [
|
||||
[
|
||||
Call(
|
||||
cannot.tool,
|
||||
{
|
||||
**cannot.args,
|
||||
"attachments": {
|
||||
**cannot.args["attachments"],
|
||||
"implementation.json": json.dumps(implementation),
|
||||
},
|
||||
},
|
||||
)
|
||||
],
|
||||
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
||||
]
|
||||
engine, runtime = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert reopened(result) == [
|
||||
(
|
||||
"target_architecture",
|
||||
(
|
||||
"Check K-2 cannot be implemented as written (no such argument). "
|
||||
"Rewrite it so it can be met, or remove it with a reason."
|
||||
),
|
||||
)
|
||||
]
|
||||
assert "the code cannot meet a Check" in {
|
||||
decision["group"] for decision in result.state["decisions"]
|
||||
}
|
||||
validations = [
|
||||
run for run in runtime.runs if run.role == specialist(K.VALIDATION_RESULTS)
|
||||
]
|
||||
assert len(validations) == 1
|
||||
|
||||
|
||||
async def test_a_check_that_fails_twice_running_is_sent_back_to_the_graph(tmp_path):
|
||||
"""The code was revised and the same Check failed again: the Check is suspect."""
|
||||
turns = once()
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 3
|
||||
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [
|
||||
[failing("K-1")],
|
||||
[failing("K-1")],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
[first, second] = reopened(result)
|
||||
assert first[0] == "deployment_guidance"
|
||||
assert second[0] == "target_architecture"
|
||||
assert "K-1 failed in two validations in a row" in second[1]
|
||||
assert actions(result).count("revise_code") == 1
|
||||
|
||||
|
||||
async def test_without_jev_a_gap_naming_a_real_requirement_revises_the_graph(tmp_path):
|
||||
turns = once()
|
||||
for kind in (
|
||||
K.TARGET_ARCHITECTURE,
|
||||
K.ARCHITECTURE_DESCRIPTION,
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
):
|
||||
turns[specialist(kind)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [
|
||||
[addition("NFR-1 has no Check on timeouts")],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
[(kind, reason)] = reopened(result)
|
||||
assert kind == "target_architecture"
|
||||
assert reason == (
|
||||
"Add a Check on E-1: A timeout is set. (serves NFR-1 has no Check on timeouts)."
|
||||
)
|
||||
|
||||
|
||||
async def test_without_jev_a_gap_that_names_nothing_real_is_left(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("it would be nice")]]
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert reopened(result) == []
|
||||
|
||||
|
||||
async def test_jev_judges_whether_a_gap_is_worth_a_revision(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("NFR-1")]]
|
||||
immaterial = FakeDecider({"gap_material": Answer(0.04, 0.92)})
|
||||
engine, _ = engine_for(tmp_path, turns, immaterial)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
# The rule alone would have revised the graph; the reading says it is not needed.
|
||||
assert reopened(result) == []
|
||||
judged = next(d for d in result.state["decisions"] if d["action"] == "proceed")
|
||||
assert judged["judged"] == {"gap_material": {"value": 0.04, "confidence": 0.92}}
|
||||
[(state, questions, scope)] = immaterial.asked
|
||||
assert "A timeout is set." in state and scope == result.execution_id
|
||||
assert questions["gap_material"].kind == "noul"
|
||||
|
||||
|
||||
async def test_an_uncertain_gap_is_put_to_the_user_who_decides(tmp_path):
|
||||
turns = once()
|
||||
for kind in (
|
||||
K.TARGET_ARCHITECTURE,
|
||||
K.ARCHITECTURE_DESCRIPTION,
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
):
|
||||
turns[specialist(kind)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [
|
||||
[addition("NFR-1")],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"gap_material": Answer(0.5, 0.0), # the grey zone
|
||||
"wants_revision": Answer(0.96, 0.92),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
|
||||
result = await engine.start("Ingest webhooks")
|
||||
for phase in (
|
||||
"requirements_discovery",
|
||||
"solution_architecture",
|
||||
"solution_implementation",
|
||||
):
|
||||
result = await engine.resume(result.execution_id, approve(phase))
|
||||
|
||||
[question] = result.prompt["questions"]
|
||||
assert (
|
||||
"found gaps in the Target architecture: A timeout is set."
|
||||
in (question["question"])
|
||||
)
|
||||
|
||||
result = await engine.resume(result.execution_id, {"reply": "Yes, revise it."})
|
||||
|
||||
assert reopened(result)[0][0] == "target_architecture"
|
||||
# Reopening the graph withdrew the design Approval, so it is asked for again.
|
||||
assert result.prompt["approval"]["phase"] == "solution_architecture"
|
||||
|
||||
|
||||
# --- the user's message: act, queue or close --------------------------------------------
|
||||
|
||||
|
||||
def to_design_approval(engine):
|
||||
async def run():
|
||||
first = await engine.start("Ingest webhooks")
|
||||
return await engine.resume(
|
||||
first.execution_id, approve("requirements_discovery")
|
||||
)
|
||||
|
||||
return run()
|
||||
|
||||
|
||||
def changes(phase: str, feedback: str) -> dict:
|
||||
return {"approval": {"phase": phase, "decision": "changes", "feedback": feedback}}
|
||||
|
||||
|
||||
async def test_a_requested_change_is_routed_to_the_deliverable_it_affects(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"change_target": Answer("design_recommendations", 0.88),
|
||||
"change_clarity": Answer(1.8, 0.8),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
|
||||
design = await to_design_approval(engine)
|
||||
result = await engine.resume(
|
||||
design.execution_id,
|
||||
changes("solution_architecture", "Use customer-managed keys on the topic."),
|
||||
)
|
||||
|
||||
assert reopened(result) == [
|
||||
("design_recommendations", "Use customer-managed keys on the topic.")
|
||||
]
|
||||
# Product selection was not touched; the design is asked for again once redone.
|
||||
assert result.state["ledger"]["deliverables"]["product_selection"]["revisions"] == 0
|
||||
assert result.prompt["approval"]["phase"] == "solution_architecture"
|
||||
[(state, questions, _)] = decider.asked
|
||||
assert "The user's message: Use customer-managed keys on the topic." in state
|
||||
assert questions["change_target"].options == (
|
||||
"requirements",
|
||||
"technical_decomposition",
|
||||
"product_selection",
|
||||
"design_recommendations",
|
||||
"target_architecture",
|
||||
)
|
||||
assert "asks_change" not in questions # a structured request is known to be one
|
||||
|
||||
|
||||
async def test_an_unclear_change_is_confirmed_with_the_user_before_anything_moves(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once()
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"change_target": Answer("product_selection", 0.31), # not relied on
|
||||
"change_clarity": Answer(1.5, 0.7),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
|
||||
design = await to_design_approval(engine)
|
||||
asked = await engine.resume(
|
||||
design.execution_id, changes("solution_architecture", "Tighten it up.")
|
||||
)
|
||||
|
||||
assert reopened(asked) == []
|
||||
[question] = asked.prompt["questions"]
|
||||
assert question["question"].startswith("Do you want product_selection changed?")
|
||||
|
||||
result = await engine.resume(
|
||||
asked.execution_id,
|
||||
{"reply": "No, change the target architecture: add a Check on timeouts."},
|
||||
)
|
||||
|
||||
[(kind, reason)] = reopened(result)
|
||||
assert kind == "target_architecture"
|
||||
assert "Tighten it up." in reason and "add a Check on timeouts" in reason
|
||||
|
||||
|
||||
async def test_a_free_text_reply_that_asks_for_a_change_is_acted_on(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.REQUIREMENTS)] = [
|
||||
[sample_submission(K.REQUIREMENTS)],
|
||||
[sample_submission(K.REQUIREMENTS)],
|
||||
]
|
||||
turns[specialist(K.TECHNICAL_DECOMPOSITION)] *= 2
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"asks_change": Answer(0.93, 0.86),
|
||||
"change_target": Answer("requirements", 0.9),
|
||||
"change_clarity": Answer(1.9, 0.8),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
result = await engine.resume(
|
||||
first.execution_id, {"reply": "Retention must be 7 years, not 30 days."}
|
||||
)
|
||||
|
||||
assert reopened(result) == [
|
||||
("requirements", "Retention must be 7 years, not 30 days.")
|
||||
]
|
||||
|
||||
|
||||
async def test_without_jev_a_change_is_acted_on_only_when_the_deliverable_is_named(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once()
|
||||
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
|
||||
design = await to_design_approval(engine)
|
||||
unnamed = await engine.resume(
|
||||
design.execution_id, changes("solution_architecture", "Make it cheaper.")
|
||||
)
|
||||
|
||||
assert reopened(unnamed) == []
|
||||
assert unnamed.prompt["questions"][0]["question"].startswith(
|
||||
"Which Deliverable should change"
|
||||
)
|
||||
|
||||
named = await engine.resume(
|
||||
unnamed.execution_id, {"reply": "The design recommendations."}
|
||||
)
|
||||
|
||||
[(kind, reason)] = reopened(named)
|
||||
assert kind == "design_recommendations"
|
||||
assert reason == (
|
||||
"Make it cheaper.\nThe user then said: The design recommendations."
|
||||
)
|
||||
|
||||
|
||||
# --- permission is the Ledger's, whatever was decided ------------------------------------
|
||||
|
||||
|
||||
async def test_past_the_revision_limit_the_user_decides(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]]
|
||||
engine, _ = engine_for(tmp_path, turns, max_revisions=1)
|
||||
|
||||
result = await engine.start("Ingest webhooks")
|
||||
for phase in (
|
||||
"requirements_discovery",
|
||||
"solution_architecture",
|
||||
"solution_implementation",
|
||||
"solution_implementation",
|
||||
):
|
||||
result = await engine.resume(result.execution_id, approve(phase))
|
||||
|
||||
assert len(reopened(result)) == 1
|
||||
[question] = result.prompt["questions"]
|
||||
assert (
|
||||
"deployment_guidance has reached its revision limit (1)"
|
||||
in (question["question"])
|
||||
)
|
||||
480
tests/execution/test_working_copy.py
Normal file
480
tests/execution/test_working_copy.py
Normal file
@@ -0,0 +1,480 @@
|
||||
"""A Specialist's working copy, and what each Specialist is given to read.
|
||||
|
||||
Three things made an Execution slow: a Specialist rewrote a whole Deliverable in one
|
||||
reply, sent every file again on each tool call, and read every earlier Deliverable in
|
||||
full. A working copy and trimmed reading remove each of them.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import (
|
||||
SAMPLE_DATA,
|
||||
Call,
|
||||
FakeAgentRuntime,
|
||||
sample_submission,
|
||||
)
|
||||
from app.adk.runtime import NoDecider, RoleSpec, Submitted, ToolResult
|
||||
from app.agent import ExecutionEngine
|
||||
from app.nodes.router import Router, Thresholds
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.tools.local import TerraformRunner, local_tools
|
||||
from app.tools.toolkit import make_tool_factory
|
||||
from app.tools.workspace import file_tools, graph_tools
|
||||
from app.workflows.deliverables import (
|
||||
DEPENDENCY_GRAPH,
|
||||
DeliverableKind,
|
||||
dependencies_for,
|
||||
)
|
||||
from app.workflows.target_architecture import parse_graph, render_tables
|
||||
|
||||
K = DeliverableKind
|
||||
GRAPH = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
UPSTREAM = {
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
||||
},
|
||||
"technical_decomposition": {
|
||||
"attachments": {"components.json": SAMPLE_DATA["components.json"]}
|
||||
},
|
||||
"product_selection": {
|
||||
"attachments": {"products.json": SAMPLE_DATA["products.json"]}
|
||||
},
|
||||
"design_recommendations": {
|
||||
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
|
||||
},
|
||||
"target_architecture": {
|
||||
"attachments": {
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def tools_of(kind, working, upstream=UPSTREAM, extra=()):
|
||||
return {
|
||||
tool.name: tool
|
||||
for tool in specialist_tools(
|
||||
kind, extra, allow_cloud_read=False, upstream=upstream, working=working
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
# --- the Target architecture is written once -------------------------------------------
|
||||
|
||||
|
||||
def test_tables_are_rendered_from_the_graph():
|
||||
tables = render_tables(parse_graph(SAMPLE_DATA["target-architecture.json"]))
|
||||
|
||||
assert "## Elements" in tables and "## Relationships" in tables
|
||||
assert (
|
||||
"| E-1 | Ingress service | Example Run | example_run_service | C-1 |" in tables
|
||||
)
|
||||
assert "| R-1 | Ingress service | runs as | Ingress identity |" in tables
|
||||
assert "| K-1 | E-1 | The service runs at least two instances. | NFR-1 |" in tables
|
||||
|
||||
|
||||
def test_a_table_cell_cannot_break_its_row():
|
||||
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
graph["checks"][0]["statement"] = "a | b\nc"
|
||||
|
||||
assert "| a \\| b c |" in render_tables(parse_graph(json.dumps(graph)))
|
||||
|
||||
|
||||
async def test_the_engine_writes_the_tables_and_the_diagram_on_submission():
|
||||
tools = tools_of(K.TARGET_ARCHITECTURE, {})
|
||||
|
||||
result = await tools["submit_deliverable"].handler(
|
||||
content="## Design rationale\nOne service, one identity.\n",
|
||||
attachments={
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
},
|
||||
)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
assert result.outcome.content.startswith("## Design rationale")
|
||||
assert "## Checks" in result.outcome.content
|
||||
assert result.outcome.attachments["architecture.mmd"].startswith("flowchart")
|
||||
|
||||
|
||||
async def test_the_graph_is_built_a_few_items_at_a_time_and_attached_on_submission():
|
||||
working: dict = {}
|
||||
tools = tools_of(K.TARGET_ARCHITECTURE, working)
|
||||
|
||||
await tools["graph_put"].handler(elements=GRAPH["elements"])
|
||||
await tools["graph_put"].handler(relationships=GRAPH["relationships"])
|
||||
put = await tools["graph_put"].handler(checks=GRAPH["checks"])
|
||||
assert put.message == (
|
||||
"Added 2, replaced 0. The graph has 2 Elements, 1 Relationships and 2 Checks."
|
||||
)
|
||||
|
||||
result = await tools["submit_deliverable"].handler(
|
||||
content="## Design rationale\nReasons.\n"
|
||||
)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
submitted = json.loads(result.outcome.attachments["target-architecture.json"])
|
||||
assert [item["id"] for item in submitted["checks"]] == ["K-1", "K-2"]
|
||||
|
||||
|
||||
async def test_a_revision_amends_the_graph_that_was_submitted():
|
||||
working: dict = {}
|
||||
tools = tools_of(K.TARGET_ARCHITECTURE, working)
|
||||
await tools["submit_deliverable"].handler(
|
||||
content="## Design rationale\nReasons.\n",
|
||||
attachments={
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
},
|
||||
)
|
||||
|
||||
# A later attempt, with the working memory the Execution kept.
|
||||
tools = tools_of(K.TARGET_ARCHITECTURE, working)
|
||||
changed = {**GRAPH["checks"][0], "statement": "min_instance_count is at least 2"}
|
||||
put = await tools["graph_put"].handler(checks=[changed])
|
||||
read = await tools["graph_read"].handler()
|
||||
|
||||
assert put.message.startswith("Added 0, replaced 1.")
|
||||
assert "min_instance_count is at least 2" in read.message
|
||||
assert len(json.loads(read.message)["checks"]) == 2
|
||||
|
||||
|
||||
async def test_graph_items_can_be_removed_and_an_item_without_an_id_is_refused():
|
||||
working: dict = {}
|
||||
tools = {tool.name: tool for tool in graph_tools(working)}
|
||||
await tools["graph_put"].handler(
|
||||
elements=GRAPH["elements"],
|
||||
not_applicable=[{"recommendation": "DR-9", "reason": "Single region."}],
|
||||
)
|
||||
|
||||
removed = await tools["graph_remove"].handler(ids=["E-2", "DR-9"])
|
||||
refused = await tools["graph_put"].handler(checks=[{"statement": "no id"}])
|
||||
|
||||
assert removed.message.startswith("Removed 2.")
|
||||
assert refused.message.startswith("Refused") and "checks[0]" in refused.message
|
||||
assert [e["id"] for e in working["graph"]["elements"]] == ["E-1"]
|
||||
|
||||
|
||||
# --- files are written once and changed in place ----------------------------------------
|
||||
|
||||
|
||||
async def test_a_file_is_written_once_and_changed_in_place():
|
||||
working: dict = {}
|
||||
tools = {tool.name: tool for tool in file_tools(working)}
|
||||
|
||||
await tools["write_file"].handler(
|
||||
path="terraform/main.tf", content="a = 1\nb = 1\n"
|
||||
)
|
||||
edited = await tools["edit_file"].handler(
|
||||
path="terraform/main.tf", old="b = 1", new="b = 2"
|
||||
)
|
||||
listing = await tools["read_file"].handler()
|
||||
|
||||
assert edited.message == "Edited terraform/main.tf."
|
||||
assert working["files"] == {"terraform/main.tf": "a = 1\nb = 2\n"}
|
||||
assert "- terraform/main.tf (2 lines)" in listing.message
|
||||
|
||||
|
||||
async def test_an_edit_that_does_not_match_exactly_once_changes_nothing():
|
||||
working: dict = {"files": {"main.tf": "x = 1\nx = 1\n"}}
|
||||
tools = {tool.name: tool for tool in file_tools(working)}
|
||||
|
||||
twice = await tools["edit_file"].handler(path="main.tf", old="x = 1", new="x = 2")
|
||||
never = await tools["edit_file"].handler(path="main.tf", old="y", new="z")
|
||||
absent = await tools["edit_file"].handler(path="other.tf", old="x", new="y")
|
||||
unsafe = await tools["write_file"].handler(path="../main.tf", content="x")
|
||||
|
||||
assert "it matches 2" in twice.message and "it matches 0" in never.message
|
||||
assert absent.message.startswith("Refused") and unsafe.message.startswith("Refused")
|
||||
assert working["files"] == {"main.tf": "x = 1\nx = 1\n"}
|
||||
|
||||
|
||||
async def test_the_working_copy_is_what_is_submitted():
|
||||
working: dict = {}
|
||||
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
|
||||
for path in ("terraform/main.tf", "terraform/variables.tf"):
|
||||
await tools["write_file"].handler(path=path, content=f"# {path}\n")
|
||||
await tools["write_file"].handler(
|
||||
path="implementation.json", content=SAMPLE_DATA["implementation.json"]
|
||||
)
|
||||
content = sample_submission(K.DEPLOYMENT_GUIDANCE).args["content"]
|
||||
|
||||
result = await tools["submit_deliverable"].handler(content=content)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
assert set(result.outcome.attachments) == {
|
||||
"terraform/main.tf",
|
||||
"terraform/variables.tf",
|
||||
"implementation.json",
|
||||
}
|
||||
|
||||
|
||||
async def test_a_file_passed_on_submission_replaces_its_working_copy():
|
||||
working: dict = {"files": {"terraform/main.tf": "old\n"}}
|
||||
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
|
||||
call = sample_submission(K.DEPLOYMENT_GUIDANCE)
|
||||
|
||||
result = await tools["submit_deliverable"].handler(**call.args)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
assert result.outcome.attachments["terraform/main.tf"] != "old\n"
|
||||
assert working["files"]["terraform/main.tf"] != "old\n"
|
||||
|
||||
|
||||
def test_only_a_specialist_that_owns_files_can_write_them():
|
||||
assert "write_file" in tools_of(K.DEPLOYMENT_GUIDANCE, {})
|
||||
assert "write_file" not in tools_of(K.VALIDATION_RESULTS, {})
|
||||
assert "graph_put" in tools_of(K.TARGET_ARCHITECTURE, {})
|
||||
assert "graph_put" not in tools_of(K.DEPLOYMENT_GUIDANCE, {})
|
||||
|
||||
|
||||
# --- Terraform runs on the working copy -------------------------------------------------
|
||||
|
||||
|
||||
class Runner(TerraformRunner):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.validated: list[dict[str, str]] = []
|
||||
|
||||
async def validate(self, files):
|
||||
self.validated.append(dict(files))
|
||||
return ToolResult("terraform validate passed:\n")
|
||||
|
||||
async def formatted(self, files):
|
||||
return {name: content.strip() + "\n" for name, content in files.items()}
|
||||
|
||||
|
||||
async def test_terraform_checks_take_no_files_when_there_is_a_working_copy():
|
||||
runner = Runner()
|
||||
files = {"terraform/main.tf": "x = 1\n", "implementation.json": "[]"}
|
||||
(validate,) = local_tools(["terraform_validate"], runner, files)
|
||||
|
||||
await validate.handler()
|
||||
|
||||
assert validate.parameters["required"] == []
|
||||
assert "pass no files" in validate.description
|
||||
# Only the Terraform files are checked.
|
||||
assert runner.validated == [{"terraform/main.tf": "x = 1\n"}]
|
||||
|
||||
|
||||
async def test_formatting_changes_the_working_copy_and_returns_only_names():
|
||||
files = {"terraform/main.tf": " x = 1 \n"}
|
||||
(fmt,) = local_tools(["terraform_fmt"], Runner(), files)
|
||||
|
||||
result = await fmt.handler()
|
||||
|
||||
assert files == {"terraform/main.tf": "x = 1\n"}
|
||||
assert result.message == (
|
||||
"terraform fmt reformatted 1 file(s) in your working copy: terraform/main.tf."
|
||||
)
|
||||
|
||||
|
||||
async def test_files_passed_anyway_are_written_to_the_working_copy_first():
|
||||
runner = Runner()
|
||||
files: dict[str, str] = {}
|
||||
(validate,) = local_tools(["terraform_validate"], runner, files)
|
||||
|
||||
await validate.handler(files=[{"path": "terraform/main.tf", "content": "y"}])
|
||||
|
||||
assert files == {"terraform/main.tf": "y"}
|
||||
assert runner.validated == [files]
|
||||
|
||||
|
||||
def roles_with(tools_by_role: dict[str, tuple[str, ...]]) -> dict[str, RoleSpec]:
|
||||
return {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(
|
||||
spec.specialist,
|
||||
"Produce.",
|
||||
budget=15,
|
||||
tools=tools_by_role.get(spec.specialist, ()),
|
||||
)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def test_the_validator_checks_the_submitted_code_without_being_sent_it(tmp_path):
|
||||
runner = Runner()
|
||||
roles = roles_with({"validator": ("terraform_validate",)})
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
turns["validator"] = [
|
||||
[Call("terraform_validate", {}), sample_submission(K.VALIDATION_RESULTS)]
|
||||
]
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
tool_factory=make_tool_factory(
|
||||
runtime,
|
||||
roles,
|
||||
{},
|
||||
local_tools=lambda working_copy: local_tools(
|
||||
["terraform_validate"], runner, working_copy
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert runner.validated == [
|
||||
{
|
||||
"terraform/main.tf": "# terraform/main.tf\n",
|
||||
"terraform/variables.tf": "# terraform/variables.tf\n",
|
||||
}
|
||||
]
|
||||
# The code is the code author's: validation attaches its verdicts and nothing else.
|
||||
validation = result.state["deliverables"]["validation_results"]
|
||||
assert set(validation["attachments"]) == {"verdicts.json"}
|
||||
|
||||
|
||||
# --- each Specialist reads what it works from -------------------------------------------
|
||||
|
||||
|
||||
def submitted() -> dict[str, dict]:
|
||||
return {
|
||||
kind.value: {
|
||||
"content": f"# {kind}\nProse.\n",
|
||||
"attachments": {
|
||||
name: SAMPLE_DATA.get(name, f"# {name}\n") for name in spec.attachments
|
||||
},
|
||||
}
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
|
||||
|
||||
def test_the_code_is_written_from_the_graph_and_the_ids_alone():
|
||||
given = dependencies_for(K.DEPLOYMENT_GUIDANCE, submitted())
|
||||
|
||||
assert given["target_architecture"] == {
|
||||
"attachments": {
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
}
|
||||
}
|
||||
assert all("content" not in deliverable for deliverable in given.values())
|
||||
assert set(given["requirements"]["attachments"]) == {"requirements.json"}
|
||||
|
||||
|
||||
def test_a_direct_dependency_is_read_whole_and_the_rest_as_data():
|
||||
given = dependencies_for(K.TARGET_ARCHITECTURE, submitted())
|
||||
|
||||
assert "content" in given["product_selection"]
|
||||
assert "content" in given["design_recommendations"]
|
||||
assert "content" not in given["requirements"]
|
||||
assert "content" not in given["technical_decomposition"]
|
||||
|
||||
|
||||
def test_the_validator_reads_the_code_whole_and_the_graph_as_data():
|
||||
given = dependencies_for(K.VALIDATION_RESULTS, submitted())
|
||||
|
||||
assert "terraform/main.tf" in given["deployment_guidance"]["attachments"]
|
||||
assert "content" not in given["target_architecture"]
|
||||
# A Deliverable with no data file has nothing to trim to, so it comes whole.
|
||||
assert "content" in given["architecture_description"]
|
||||
|
||||
|
||||
def test_the_solution_guide_is_assembled_from_every_deliverable_whole():
|
||||
given = dependencies_for(K.SOLUTION_GUIDE, submitted())
|
||||
|
||||
assert len(given) == len(DEPENDENCY_GRAPH) - 1
|
||||
assert all("content" in deliverable for deliverable in given.values())
|
||||
|
||||
|
||||
async def test_a_specialist_is_told_what_its_working_copy_already_holds(tmp_path):
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
failing = sample_submission(K.VALIDATION_RESULTS)
|
||||
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
verdicts["checks"][0].update(verdict="fail", evidence="min instances is 1")
|
||||
turns["validator"] = [
|
||||
[
|
||||
Call(
|
||||
failing.tool,
|
||||
{
|
||||
**failing.args,
|
||||
"attachments": {"verdicts.json": json.dumps(verdicts)},
|
||||
},
|
||||
)
|
||||
],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
turns["iac-author"] = [[sample_submission(K.DEPLOYMENT_GUIDANCE)]] * 2
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=roles_with({}),
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
|
||||
await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
first, second = (run for run in runtime.runs if run.role == "iac-author")
|
||||
assert first.task.context["working_copy"] is None
|
||||
assert second.task.context["working_copy"] == {
|
||||
"files": ["implementation.json", "terraform/main.tf", "terraform/variables.tf"]
|
||||
}
|
||||
|
||||
|
||||
async def test_a_deliverable_given_as_data_can_be_read_whole_on_demand():
|
||||
library = {
|
||||
"requirements": {
|
||||
"content": "# requirements\nProse.\n",
|
||||
"attachments": {"requirements.json": "[]"},
|
||||
}
|
||||
}
|
||||
tools = {
|
||||
tool.name: tool
|
||||
for tool in specialist_tools(
|
||||
K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False, library=library
|
||||
)
|
||||
}
|
||||
|
||||
read = await tools["read_deliverable"].handler(kind="requirements")
|
||||
refused = await tools["read_deliverable"].handler(kind="solution_guide")
|
||||
|
||||
assert (
|
||||
read.message.startswith("# requirements")
|
||||
and "requirements.json" in read.message
|
||||
)
|
||||
assert refused.message.startswith("Refused")
|
||||
# With nothing trimmed there is nothing to read back.
|
||||
assert "read_deliverable" not in tools_of(K.REQUIREMENTS, {})
|
||||
|
||||
|
||||
async def test_the_engine_offers_the_trimmed_deliverables_for_reading(tmp_path):
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=roles_with({}),
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
|
||||
await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
author = next(run for run in runtime.runs if run.role == "iac-author")
|
||||
packager = next(run for run in runtime.runs if run.role == "packager")
|
||||
assert "read_deliverable" in author.tools
|
||||
assert "content" not in author.task.context["dependencies"]["requirements"]
|
||||
assert "read_deliverable" not in packager.tools
|
||||
Reference in New Issue
Block a user