feat: establish the Target architecture as a graph before code
Design is now approved before any infrastructure code is written (ADR-0005). A new target-architect Specialist produces one graph of Elements, Relationships and Checks; the diagram is rendered from it, deployment guidance must account for every Check, and validation gives a verdict per Check. Requirements, components, products and recommendations carry ids in data files, and a submission with a gap in the Requirements chain is refused. Also in this commit: - Customer cases group Executions, with an intent history per case - lesson candidates are captured when an Execution ends - evaluation gains chain, verdict, judged and interactive checks, and per-case thresholds - Terraform providers are cached between checks - the requirements analyst proposes defaults instead of asking - the legacy-baseline case and the checks that only read it are removed - ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed - plans for iterations 4 to 7 under .agents/plans/ Unproven with a real model: no evaluation run has completed on this design. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,13 +3,14 @@
|
||||
Driven entirely through ExecutionEngine.start/resume with a scripted agent runtime.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
|
||||
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
|
||||
from app.agent import ExecutionEngine
|
||||
from app.recorder import InMemoryRecorder
|
||||
@@ -50,12 +51,7 @@ def reopen(kind: DeliverableKind, reason: str) -> Call:
|
||||
FINISH = Call("finish")
|
||||
|
||||
|
||||
def submission(kind: DeliverableKind, note: str = "") -> Call:
|
||||
spec = DEPENDENCY_GRAPH[kind]
|
||||
body = "\n".join(f"## {section}\n{note or 'Details.'}" for section in spec.sections)
|
||||
content = f"# {kind}\n{note or 'Summary.'}\n{body}\n"
|
||||
attachments = {name: f"# {name}\n" for name in spec.attachments}
|
||||
return Call("submit_deliverable", {"content": content, "attachments": attachments})
|
||||
submission = sample_submission
|
||||
|
||||
|
||||
def submits(kind: DeliverableKind) -> list[Call]:
|
||||
@@ -84,9 +80,16 @@ def happy_path_turns(**extra_specialist_turns) -> dict:
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)],
|
||||
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.TARGET_ARCHITECTURE)],
|
||||
[
|
||||
request_approval(P.SOLUTION_ARCHITECTURE),
|
||||
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[
|
||||
request_approval(P.SOLUTION_IMPLEMENTATION),
|
||||
delegate(K.VALIDATION_RESULTS),
|
||||
],
|
||||
[request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
|
||||
[request_approval(P.PACKAGING), FINISH],
|
||||
],
|
||||
@@ -112,7 +115,14 @@ async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path
|
||||
assert set(ledger["phases"].values()) == {"approved"}
|
||||
assert {d["status"] for d in ledger["deliverables"].values()} == {"submitted"}
|
||||
|
||||
out = tmp_path / "deliverables" / "executions" / result.execution_id
|
||||
out = (
|
||||
tmp_path
|
||||
/ "deliverables"
|
||||
/ "cases"
|
||||
/ "unassigned"
|
||||
/ "executions"
|
||||
/ result.execution_id
|
||||
)
|
||||
assert (out / "solution-architecture-guide.md").is_file()
|
||||
for kind in K:
|
||||
assert (out / "deliverables" / f"{kind}.md").is_file(), kind
|
||||
@@ -122,7 +132,7 @@ async def test_pre_approved_execution_completes_and_writes_deliverables(tmp_path
|
||||
assert recorder.statuses[result.execution_id][-1]["status"] == "completed"
|
||||
assert {s["kind"] for s in recorder.submissions[result.execution_id]} == set(K)
|
||||
recorded_types = [e.type for e in recorder.events[result.execution_id]]
|
||||
assert recorded_types.count("approved") == 4
|
||||
assert recorded_types.count("approved") == 5
|
||||
|
||||
|
||||
async def test_independent_specialists_run_together_between_orchestrator_turns(
|
||||
@@ -133,12 +143,12 @@ async def test_independent_specialists_run_together_between_orchestrator_turns(
|
||||
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
|
||||
|
||||
roles = [run.role for run in runtime.runs]
|
||||
fourth_turn = [i for i, role in enumerate(roles) if role == "orchestrator"][3]
|
||||
assert set(roles[fourth_turn + 1 : fourth_turn + 3]) == {
|
||||
specialist(K.ARCHITECTURE_DIAGRAM),
|
||||
specialist(K.DESIGN_RECOMMENDATIONS),
|
||||
sixth_turn = [i for i, role in enumerate(roles) if role == "orchestrator"][5]
|
||||
assert set(roles[sixth_turn + 1 : sixth_turn + 3]) == {
|
||||
specialist(K.ARCHITECTURE_DESCRIPTION),
|
||||
specialist(K.DEPLOYMENT_GUIDANCE),
|
||||
}
|
||||
assert roles[fourth_turn + 3] == "orchestrator"
|
||||
assert roles[sixth_turn + 3] == "orchestrator"
|
||||
|
||||
|
||||
async def test_specialist_receives_its_dependencies_content(tmp_path):
|
||||
@@ -154,9 +164,13 @@ async def test_specialist_receives_its_dependencies_content(tmp_path):
|
||||
"requirements",
|
||||
"technical_decomposition",
|
||||
"product_selection",
|
||||
"architecture_diagram",
|
||||
"design_recommendations",
|
||||
"target_architecture",
|
||||
]
|
||||
assert "architecture.mmd" in dependencies["architecture_diagram"]["attachments"]
|
||||
# The engine drew the diagram from the graph when the Target architecture was submitted.
|
||||
assert dependencies["target_architecture"]["attachments"][
|
||||
"architecture.mmd"
|
||||
].startswith("flowchart")
|
||||
assert describer_run.task.deliverable == "architecture_description"
|
||||
|
||||
|
||||
@@ -334,16 +348,17 @@ async def test_requested_changes_lead_to_a_revision_and_a_new_approval(tmp_path)
|
||||
async def test_questions_from_parallel_specialists_reach_the_user_in_one_prompt(
|
||||
tmp_path,
|
||||
):
|
||||
def diagrammer_turn(task):
|
||||
def describer_turn(task):
|
||||
if task.context["answers"]:
|
||||
return submits(K.ARCHITECTURE_DIAGRAM)
|
||||
return [Call("needs_input", {"questions": ["Single region or multi-region?"]})]
|
||||
return submits(K.ARCHITECTURE_DESCRIPTION)
|
||||
return [Call("needs_input", {"questions": ["Who reads this description?"]})]
|
||||
|
||||
def advisor_turn(task):
|
||||
def iac_turn(task):
|
||||
if task.context["answers"]:
|
||||
return submits(K.DESIGN_RECOMMENDATIONS)
|
||||
return [Call("needs_input", {"questions": ["What is the RTO?"]})]
|
||||
return submits(K.DEPLOYMENT_GUIDANCE)
|
||||
return [Call("needs_input", {"questions": ["Which region?"]})]
|
||||
|
||||
both = delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
@@ -351,47 +366,59 @@ async def test_questions_from_parallel_specialists_reach_the_user_in_one_prompt(
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[request_approval(P.REQUIREMENTS_DISCOVERY)],
|
||||
[delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.TARGET_ARCHITECTURE)],
|
||||
[request_approval(P.SOLUTION_ARCHITECTURE)],
|
||||
[both],
|
||||
[ask_user()],
|
||||
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
|
||||
[both],
|
||||
[ask_user("Continue?")],
|
||||
],
|
||||
specialist(K.REQUIREMENTS): [submits(K.REQUIREMENTS)],
|
||||
specialist(K.TECHNICAL_DECOMPOSITION): [submits(K.TECHNICAL_DECOMPOSITION)],
|
||||
specialist(K.PRODUCT_SELECTION): [submits(K.PRODUCT_SELECTION)],
|
||||
specialist(K.ARCHITECTURE_DIAGRAM): [diagrammer_turn, diagrammer_turn],
|
||||
specialist(K.DESIGN_RECOMMENDATIONS): [advisor_turn, advisor_turn],
|
||||
**{
|
||||
specialist(kind): [submits(kind)]
|
||||
for kind in (
|
||||
K.REQUIREMENTS,
|
||||
K.TECHNICAL_DECOMPOSITION,
|
||||
K.PRODUCT_SELECTION,
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
K.TARGET_ARCHITECTURE,
|
||||
)
|
||||
},
|
||||
specialist(K.ARCHITECTURE_DESCRIPTION): [describer_turn, describer_turn],
|
||||
specialist(K.DEPLOYMENT_GUIDANCE): [iac_turn, iac_turn],
|
||||
}
|
||||
)
|
||||
engine = make_engine(tmp_path, runtime)
|
||||
approval = await engine.start("Ingestion")
|
||||
first = await engine.resume(
|
||||
design = await engine.resume(
|
||||
approval.execution_id,
|
||||
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
|
||||
)
|
||||
first = await engine.resume(
|
||||
design.execution_id,
|
||||
{"approval": {"phase": "solution_architecture", "decision": "approve"}},
|
||||
)
|
||||
|
||||
assert first.prompt == {
|
||||
"questions": [
|
||||
{
|
||||
"from": "architecture_diagram",
|
||||
"question": "Single region or multi-region?",
|
||||
"from": "architecture_description",
|
||||
"question": "Who reads this description?",
|
||||
},
|
||||
{"from": "design_recommendations", "question": "What is the RTO?"},
|
||||
{"from": "deployment_guidance", "question": "Which region?"},
|
||||
]
|
||||
}
|
||||
|
||||
second = await engine.resume(
|
||||
first.execution_id, {"reply": "Single region, RTO 1 hour"}
|
||||
)
|
||||
second = await engine.resume(first.execution_id, {"reply": "Engineers, in London"})
|
||||
|
||||
deliverables = second.state["ledger"]["deliverables"]
|
||||
assert deliverables["architecture_diagram"]["status"] == "submitted"
|
||||
assert deliverables["design_recommendations"]["status"] == "submitted"
|
||||
advisor_retry = [
|
||||
r for r in runtime.runs if r.role == specialist(K.DESIGN_RECOMMENDATIONS)
|
||||
assert deliverables["architecture_description"]["status"] == "submitted"
|
||||
assert deliverables["deployment_guidance"]["status"] == "submitted"
|
||||
iac_retry = [
|
||||
r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)
|
||||
][1]
|
||||
assert advisor_retry.task.context["answers"] == [
|
||||
{"questions": ["What is the RTO?"], "reply": "Single region, RTO 1 hour"}
|
||||
assert iac_retry.task.context["answers"] == [
|
||||
{"questions": ["Which region?"], "reply": "Engineers, in London"}
|
||||
]
|
||||
|
||||
|
||||
@@ -404,14 +431,18 @@ async def test_validation_failure_revises_deployment_guidance_and_reapproves(tmp
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[request_approval(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.ARCHITECTURE_DIAGRAM, K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE)],
|
||||
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.TARGET_ARCHITECTURE)],
|
||||
[
|
||||
request_approval(P.SOLUTION_ARCHITECTURE),
|
||||
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[request_approval(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
||||
[
|
||||
reopen(K.DEPLOYMENT_GUIDANCE, "terraform plan: missing IAM binding"),
|
||||
delegate(K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[request_approval(P.SOLUTION_ARCHITECTURE), delegate(K.VALIDATION_RESULTS)],
|
||||
[request_approval(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
||||
[request_approval(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
|
||||
[request_approval(P.PACKAGING), FINISH],
|
||||
]
|
||||
@@ -430,9 +461,10 @@ async def test_validation_failure_revises_deployment_guidance_and_reapproves(tmp
|
||||
assert result.status == "completed"
|
||||
events = result.state["ledger"]["events"]
|
||||
withdrawn = [e["subject"] for e in events if e["type"] == "approval_withdrawn"]
|
||||
assert withdrawn == ["solution_architecture"]
|
||||
assert withdrawn == ["solution_implementation"]
|
||||
requests = [e["subject"] for e in events if e["type"] == "approval_requested"]
|
||||
assert requests.count("solution_architecture") == 2
|
||||
assert requests.count("solution_implementation") == 2
|
||||
assert requests.count("solution_architecture") == 1
|
||||
iac_runs = [r for r in runtime.runs if r.role == specialist(K.DEPLOYMENT_GUIDANCE)]
|
||||
assert iac_runs[1].task.context["feedback"] == [
|
||||
"terraform plan: missing IAM binding"
|
||||
@@ -456,7 +488,12 @@ async def test_pre_approved_escalation_ends_with_warnings(tmp_path):
|
||||
|
||||
assert result.status == "completed_with_warnings"
|
||||
assert "revision limit" in result.state["warnings"][0]
|
||||
assert not (tmp_path / "deliverables").exists()
|
||||
# No Deliverable files are written, but the request stays on record.
|
||||
case = tmp_path / "deliverables" / "cases" / "unassigned"
|
||||
assert not (case / "executions").exists()
|
||||
[intent] = json.loads((case / "intent" / "history.json").read_text())["intents"]
|
||||
assert intent["status"] == "completed_with_warnings"
|
||||
assert intent["execution"] is None
|
||||
|
||||
|
||||
# --- budgets -------------------------------------------------------------------------------------
|
||||
@@ -514,7 +551,7 @@ def cloud_read_factory(calls: list[str]):
|
||||
tier="cloud_read",
|
||||
)
|
||||
|
||||
def factory(role: RoleSpec, deliverable: str | None):
|
||||
def factory(role: RoleSpec, deliverable: str | None, working: dict):
|
||||
return [tool] if role.name == "requirements-analyst" else []
|
||||
|
||||
return factory
|
||||
@@ -686,3 +723,249 @@ async def test_pre_approved_questions_use_matching_scripted_answers(tmp_path):
|
||||
assert "No user is available" not in first_reply
|
||||
second_reply = result.state["conversation"][4]["text"]
|
||||
assert second_reply.startswith("No user is available")
|
||||
|
||||
|
||||
# --- contract and draft in the Specialist's context (iteration 4, steps 2 and 4) -------
|
||||
|
||||
|
||||
async def test_specialist_context_carries_its_contract(tmp_path):
|
||||
runtime = FakeAgentRuntime(happy_path_turns())
|
||||
|
||||
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
|
||||
|
||||
run = next(r for r in runtime.runs if r.role == specialist(K.VALIDATION_RESULTS))
|
||||
assert run.task.context["contract"] == {
|
||||
"sections": ["Deployment dry-run", "Architecture and policy analysis"],
|
||||
"attachments": ["verdicts.json"],
|
||||
}
|
||||
assert run.task.context["draft"] is None
|
||||
|
||||
|
||||
async def test_a_draft_sent_with_questions_comes_back_with_the_answer(tmp_path):
|
||||
draft = sample_submission(K.REQUIREMENTS, note="Drafted for approval.").args
|
||||
|
||||
def confirm_then_submit_what_came_back(task):
|
||||
returned = task.context["draft"]
|
||||
return [Call("submit_deliverable", returned)]
|
||||
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user()],
|
||||
[delegate(K.REQUIREMENTS)],
|
||||
[ask_user("Anything else?")],
|
||||
],
|
||||
specialist(K.REQUIREMENTS): [
|
||||
[
|
||||
Call(
|
||||
"needs_input",
|
||||
{
|
||||
"questions": ["Do you approve this draft?"],
|
||||
"draft": draft["content"],
|
||||
"attachments": [
|
||||
{"path": path, "content": content}
|
||||
for path, content in draft["attachments"].items()
|
||||
],
|
||||
},
|
||||
)
|
||||
],
|
||||
confirm_then_submit_what_came_back,
|
||||
],
|
||||
}
|
||||
)
|
||||
engine = make_engine(tmp_path, runtime)
|
||||
|
||||
paused = await engine.start("Ingestion")
|
||||
assert paused.state["drafts"]["requirements"]["content"] == draft["content"]
|
||||
|
||||
after = await engine.resume(paused.execution_id, {"reply": "Approved."})
|
||||
|
||||
second_run = [r for r in runtime.runs if r.role == specialist(K.REQUIREMENTS)][1]
|
||||
assert second_run.task.context["draft"] == {
|
||||
"content": draft["content"],
|
||||
"attachments": draft["attachments"],
|
||||
}
|
||||
assert second_run.task.context["answers"][-1]["reply"] == "Approved."
|
||||
assert after.state["deliverables"]["requirements"]["content"] == draft["content"]
|
||||
assert after.state["drafts"] == {}
|
||||
|
||||
|
||||
# --- cap on parallel Specialists (iteration 4, step 5) ---------------------------------
|
||||
|
||||
|
||||
async def test_parallel_specialists_never_exceed_the_cap(tmp_path):
|
||||
import asyncio
|
||||
|
||||
from app.adk.runtime import RunLimiter
|
||||
|
||||
in_flight = 0
|
||||
peak = 0
|
||||
|
||||
class SlowRuntime(FakeAgentRuntime):
|
||||
async def run(self, role, task, tools):
|
||||
nonlocal in_flight, peak
|
||||
if role.name == "orchestrator":
|
||||
return await super().run(role, task, tools)
|
||||
in_flight += 1
|
||||
peak = max(peak, in_flight)
|
||||
await asyncio.sleep(0.01)
|
||||
try:
|
||||
return await super().run(role, task, tools)
|
||||
finally:
|
||||
in_flight -= 1
|
||||
|
||||
runtime = SlowRuntime(happy_path_turns())
|
||||
engine = make_engine(tmp_path, runtime, limiter=RunLimiter(1))
|
||||
|
||||
result = await engine.start("Ingestion", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert peak == 1
|
||||
|
||||
|
||||
async def test_without_a_cap_independent_specialists_overlap(tmp_path):
|
||||
import asyncio
|
||||
|
||||
in_flight = 0
|
||||
peak = 0
|
||||
|
||||
class SlowRuntime(FakeAgentRuntime):
|
||||
async def run(self, role, task, tools):
|
||||
nonlocal in_flight, peak
|
||||
if role.name == "orchestrator":
|
||||
return await super().run(role, task, tools)
|
||||
in_flight += 1
|
||||
peak = max(peak, in_flight)
|
||||
await asyncio.sleep(0.01)
|
||||
try:
|
||||
return await super().run(role, task, tools)
|
||||
finally:
|
||||
in_flight -= 1
|
||||
|
||||
runtime = SlowRuntime(happy_path_turns())
|
||||
|
||||
await make_engine(tmp_path, runtime).start("Ingestion", pre_approved=True)
|
||||
|
||||
assert peak == 2
|
||||
|
||||
|
||||
# --- Customer cases group Executions ---------------------------------------------------
|
||||
|
||||
|
||||
async def test_an_execution_in_a_customer_case_writes_under_that_case(tmp_path):
|
||||
recorder = InMemoryRecorder()
|
||||
engine = make_engine(
|
||||
tmp_path, FakeAgentRuntime(happy_path_turns()), recorder=recorder
|
||||
)
|
||||
|
||||
result = await engine.start(
|
||||
"Ingestion", pre_approved=True, customer_case="acme-webhooks"
|
||||
)
|
||||
|
||||
case_dir = tmp_path / "deliverables" / "cases" / "acme-webhooks"
|
||||
out = case_dir / "executions" / result.execution_id
|
||||
assert (out / "solution-architecture-guide.md").is_file()
|
||||
assert result.state["customer_case"] == "acme-webhooks"
|
||||
|
||||
manifest = json.loads((out / "execution.json").read_text(encoding="utf-8"))
|
||||
assert manifest["customer_case"] == "acme-webhooks"
|
||||
assert manifest["workflow_request"] == "Ingestion"
|
||||
assert manifest["intent_version"] == 1
|
||||
assert manifest["solution_guide"] == "solution-architecture-guide.md"
|
||||
assert manifest["deliverables"]["deployment_guidance"] == {
|
||||
"document": "deliverables/deployment_guidance.md",
|
||||
"attachments": [
|
||||
"implementation.json",
|
||||
"terraform/main.tf",
|
||||
"terraform/variables.tf",
|
||||
],
|
||||
}
|
||||
for entry in manifest["deliverables"].values():
|
||||
assert (out / entry["document"]).is_file()
|
||||
assert all((out / name).is_file() for name in entry["attachments"])
|
||||
assert (
|
||||
recorder.statuses[result.execution_id][-1]["details"]["customer_case"]
|
||||
== "acme-webhooks"
|
||||
)
|
||||
lessons = (
|
||||
tmp_path
|
||||
/ "lessons"
|
||||
/ "candidates"
|
||||
/ "acme-webhooks"
|
||||
/ f"{result.execution_id}.json"
|
||||
)
|
||||
assert lessons.is_file()
|
||||
assert recorder.lessons[result.execution_id]["customer_case"] == "acme-webhooks"
|
||||
|
||||
|
||||
async def test_two_executions_of_one_customer_case_sit_side_by_side(tmp_path):
|
||||
first = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
|
||||
"Ingestion", pre_approved=True, customer_case="acme-webhooks"
|
||||
)
|
||||
second = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
|
||||
"Ingestion, revised", pre_approved=True, customer_case="acme-webhooks"
|
||||
)
|
||||
|
||||
case = tmp_path / "deliverables" / "cases" / "acme-webhooks"
|
||||
assert {path.name for path in (case / "executions").iterdir()} == {
|
||||
first.execution_id,
|
||||
second.execution_id,
|
||||
}
|
||||
history = json.loads((case / "intent" / "history.json").read_text(encoding="utf-8"))
|
||||
assert history["customer_case"] == "acme-webhooks"
|
||||
assert [
|
||||
(i["version"], i["workflow_request"], i["status"], i["execution"])
|
||||
for i in history["intents"]
|
||||
] == [
|
||||
(1, "Ingestion", "completed", f"executions/{first.execution_id}"),
|
||||
(2, "Ingestion, revised", "completed", f"executions/{second.execution_id}"),
|
||||
]
|
||||
assert "Ingestion, revised" in (case / "intent" / "002.md").read_text()
|
||||
assert first.state["intent_version"] == 1 and second.state["intent_version"] == 2
|
||||
# A Customer case groups Executions; it never feeds one into the next.
|
||||
assert second.state["conversation"][0]["text"] == "Ingestion, revised"
|
||||
assert "deliverables" in second.state and first.state != second.state
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad", ["../etc", "Acme", "acme/webhooks", "", "-acme", "a" * 65, 7]
|
||||
)
|
||||
async def test_a_customer_case_that_is_not_a_safe_slug_is_refused(tmp_path, bad):
|
||||
engine = make_engine(tmp_path, FakeAgentRuntime(happy_path_turns()))
|
||||
|
||||
with pytest.raises(ValueError, match="customer_case"):
|
||||
await engine.start("Ingestion", pre_approved=True, customer_case=bad)
|
||||
|
||||
assert not (tmp_path / "deliverables").exists()
|
||||
|
||||
|
||||
async def test_an_execution_without_a_customer_case_is_grouped_as_unassigned(tmp_path):
|
||||
result = await make_engine(tmp_path, FakeAgentRuntime(happy_path_turns())).start(
|
||||
"Ingestion", pre_approved=True
|
||||
)
|
||||
|
||||
assert result.state["customer_case"] == "unassigned"
|
||||
assert [path.name for path in (tmp_path / "deliverables").iterdir()] == ["cases"]
|
||||
assert [path.name for path in (tmp_path / "deliverables" / "cases").iterdir()] == [
|
||||
"unassigned"
|
||||
]
|
||||
|
||||
|
||||
async def test_the_intent_is_on_record_as_soon_as_an_execution_starts(tmp_path):
|
||||
runtime = FakeAgentRuntime(
|
||||
{"orchestrator": [[ask_user("Which region should this run in?")]]}
|
||||
)
|
||||
engine = make_engine(tmp_path, runtime)
|
||||
|
||||
paused = await engine.start("Ingest signed webhooks", customer_case="acme-webhooks")
|
||||
|
||||
assert paused.status == "awaiting_user"
|
||||
intent = tmp_path / "deliverables" / "cases" / "acme-webhooks" / "intent"
|
||||
statement = (intent / "001.md").read_text(encoding="utf-8")
|
||||
assert "Ingest signed webhooks" in statement
|
||||
assert paused.execution_id in statement
|
||||
[entry] = json.loads((intent / "history.json").read_text())["intents"]
|
||||
assert entry["status"] == "in_progress"
|
||||
assert entry["statement"] == "intent/001.md"
|
||||
assert entry["execution_id"] == paused.execution_id
|
||||
|
||||
Reference in New Issue
Block a user