Files
orchestration_engine/tests/execution/test_lessons.py

233 lines
7.6 KiB
Python

"""Lesson candidates: what an Execution keeps about what went wrong in it."""
import json
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.runtime import RoleSpec
from execution_engine.agent import ExecutionEngine
from execution_engine.recorder import InMemoryRecorder
from execution_engine.states.lessons import refusal
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
WORKFLOW,
DeliverableKind,
PhaseId,
sample_submission,
)
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
K = DeliverableKind
P = PhaseId
ROLES = {
"orchestrator": RoleSpec(
"orchestrator", "Coordinate.", budget=40, skills=("solution-architecture",)
),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
FAILED = {"check": "K-2", "evidence": "The identity is shared with another service."}
ADDITION = {
"statement": "The service sets a request timeout.",
"applies_to": "E-1",
"reason": "NFR-1 has no Check on timeouts",
}
def delegate(*kinds: DeliverableKind) -> Call:
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
def approve(phase: PhaseId) -> Call:
return Call("request_approval", {"phase": phase.value, "summary": "done"})
def specialist(kind: DeliverableKind) -> str:
return DEPENDENCY_GRAPH[kind].specialist
def failing_validation() -> Call:
call = sample_submission(K.VALIDATION_RESULTS)
verdicts = json.loads(call.args["attachments"]["verdicts.json"])
verdicts["checks"][1] = {**FAILED, "verdict": "fail"}
verdicts["proposed_additions"] = [ADDITION]
return Call(
call.tool,
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
)
def turns_with_one_revision() -> dict:
once = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
return {
**once,
"orchestrator": [
# The first call is refused: requirements are not done yet.
[delegate(K.TECHNICAL_DECOMPOSITION), delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
[
approve(P.SOLUTION_ARCHITECTURE),
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
],
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[
Call(
"reopen",
{"kind": "deployment_guidance", "reason": "Fix the ingress block."},
),
delegate(K.DEPLOYMENT_GUIDANCE),
],
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[approve(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
[approve(P.PACKAGING), Call("finish")],
],
specialist(K.DEPLOYMENT_GUIDANCE): [
[
Call("submit_deliverable", {"content": "No headings yet."}),
sample_submission(K.DEPLOYMENT_GUIDANCE),
],
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
],
specialist(K.VALIDATION_RESULTS): [
[failing_validation()],
[sample_submission(K.VALIDATION_RESULTS)],
],
}
async def run(tmp_path, turns, **options):
recorder = InMemoryRecorder()
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=FakeAgentRuntime(turns),
roles=ROLES,
recorder=recorder,
checkpointer=InMemorySaver(),
output_root=tmp_path,
skill_versions={"solution-architecture": "abc123"},
**options,
)
result = await engine.start("Ingestion", pre_approved=True)
return result, recorder
def of_type(record: dict, kind: str) -> list[dict]:
return [c for c in record["candidates"] if c["type"] == kind]
async def test_a_finished_execution_records_what_went_wrong(tmp_path):
result, recorder = await run(tmp_path, turns_with_one_revision())
assert result.status == "completed"
record = recorder.lessons[result.execution_id]
assert of_type(record, "revision") == [
{
"type": "revision",
"deliverable": "deployment_guidance",
"reason": "Fix the ingress block.",
"requested_by": "orchestrator",
}
]
assert of_type(record, "failed_check") == [
{"type": "failed_check", "attempt": 1, **FAILED}
]
assert of_type(record, "proposed_addition") == [
{"type": "proposed_addition", "attempt": 1, **ADDITION}
]
refusals = of_type(record, "refusal")
assert {(r["role"], r["tool"], r["deliverable"]) for r in refusals} == {
("orchestrator", "delegate", None),
(
specialist(K.DEPLOYMENT_GUIDANCE),
"submit_deliverable",
"deployment_guidance",
),
}
assert all(r["message"].startswith(("Refused", "Not submitted")) for r in refusals)
async def test_the_record_names_the_execution_and_the_skills_loaded(tmp_path):
result, recorder = await run(tmp_path, turns_with_one_revision())
record = recorder.lessons[result.execution_id]
assert record["execution_id"] == result.execution_id
assert record["status"] == "completed"
assert record["role_skills"]["orchestrator"] == ["solution-architecture"]
assert record["skill_versions"] == {"solution-architecture": "abc123"}
async def test_the_record_is_also_written_beside_the_deliverables_tree(tmp_path):
result, recorder = await run(tmp_path, turns_with_one_revision())
path = (
tmp_path
/ "lessons"
/ "candidates"
/ "unassigned"
/ f"{result.execution_id}.json"
)
assert (
json.loads(path.read_text(encoding="utf-8"))
== (recorder.lessons[result.execution_id])
)
async def test_an_execution_that_ends_with_warnings_still_records_lessons(tmp_path):
turns = {
"orchestrator": [[delegate(K.REQUIREMENTS)], []],
specialist(K.REQUIREMENTS): [[]], # stops without submitting
}
result, recorder = await run(tmp_path, turns)
assert result.status == "completed_with_warnings"
record = recorder.lessons[result.execution_id]
assert of_type(record, "unfinished") == [
{
"type": "unfinished",
"deliverable": "requirements",
"role": specialist(K.REQUIREMENTS),
"reason": "stopped",
}
]
# Only the scope had been produced, so only the scope was written.
written = {path.name for path in tmp_path.rglob("deliverables/*.md")}
assert written == {"scope.md"}
def test_a_cloud_read_refusal_keeps_only_its_leading_words():
"""What a cloud-read tool says may describe the user's environment."""
item = refusal(
"requirements-analyst",
"requirements",
"compute__list_instances",
"cloud_read",
"Refused: project acme-prod-1234 does not have the API enabled.",
)
assert item["tool"] == "compute__list_instances"
assert item["message"] == "Refused"
assert "acme-prod-1234" not in item["message"]
def test_refusal_messages_are_bounded():
item = refusal(
"validator", "validation_results", "terraform_validate", "general", "x" * 900
)
assert len(item["message"]) == 500