feat: working orhcestration engine
This commit is contained in:
232
tests/execution/test_lessons.py
Normal file
232
tests/execution/test_lessons.py
Normal file
@@ -0,0 +1,232 @@
|
||||
"""Lesson candidates: what an Execution keeps about what went wrong in it."""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from execution_engine.adk.runtime import RoleSpec
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from execution_engine.states.lessons import refusal
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
WORKFLOW,
|
||||
DeliverableKind,
|
||||
PhaseId,
|
||||
sample_submission,
|
||||
)
|
||||
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
|
||||
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
|
||||
|
||||
K = DeliverableKind
|
||||
P = PhaseId
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec(
|
||||
"orchestrator", "Coordinate.", budget=40, skills=("solution-architecture",)
|
||||
),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
FAILED = {"check": "K-2", "evidence": "The identity is shared with another service."}
|
||||
ADDITION = {
|
||||
"statement": "The service sets a request timeout.",
|
||||
"applies_to": "E-1",
|
||||
"reason": "NFR-1 has no Check on timeouts",
|
||||
}
|
||||
|
||||
|
||||
def delegate(*kinds: DeliverableKind) -> Call:
|
||||
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
||||
|
||||
|
||||
def approve(phase: PhaseId) -> Call:
|
||||
return Call("request_approval", {"phase": phase.value, "summary": "done"})
|
||||
|
||||
|
||||
def specialist(kind: DeliverableKind) -> str:
|
||||
return DEPENDENCY_GRAPH[kind].specialist
|
||||
|
||||
|
||||
def failing_validation() -> Call:
|
||||
call = sample_submission(K.VALIDATION_RESULTS)
|
||||
verdicts = json.loads(call.args["attachments"]["verdicts.json"])
|
||||
verdicts["checks"][1] = {**FAILED, "verdict": "fail"}
|
||||
verdicts["proposed_additions"] = [ADDITION]
|
||||
return Call(
|
||||
call.tool,
|
||||
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
|
||||
)
|
||||
|
||||
|
||||
def turns_with_one_revision() -> dict:
|
||||
once = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
return {
|
||||
**once,
|
||||
"orchestrator": [
|
||||
# The first call is refused: requirements are not done yet.
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION), delegate(K.REQUIREMENTS)],
|
||||
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
||||
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
||||
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
||||
[delegate(K.TARGET_ARCHITECTURE)],
|
||||
[
|
||||
approve(P.SOLUTION_ARCHITECTURE),
|
||||
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
||||
[
|
||||
Call(
|
||||
"reopen",
|
||||
{"kind": "deployment_guidance", "reason": "Fix the ingress block."},
|
||||
),
|
||||
delegate(K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
||||
[approve(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
|
||||
[approve(P.PACKAGING), Call("finish")],
|
||||
],
|
||||
specialist(K.DEPLOYMENT_GUIDANCE): [
|
||||
[
|
||||
Call("submit_deliverable", {"content": "No headings yet."}),
|
||||
sample_submission(K.DEPLOYMENT_GUIDANCE),
|
||||
],
|
||||
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
||||
],
|
||||
specialist(K.VALIDATION_RESULTS): [
|
||||
[failing_validation()],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
async def run(tmp_path, turns, **options):
|
||||
recorder = InMemoryRecorder()
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=FakeAgentRuntime(turns),
|
||||
roles=ROLES,
|
||||
recorder=recorder,
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
skill_versions={"solution-architecture": "abc123"},
|
||||
**options,
|
||||
)
|
||||
result = await engine.start("Ingestion", pre_approved=True)
|
||||
return result, recorder
|
||||
|
||||
|
||||
def of_type(record: dict, kind: str) -> list[dict]:
|
||||
return [c for c in record["candidates"] if c["type"] == kind]
|
||||
|
||||
|
||||
async def test_a_finished_execution_records_what_went_wrong(tmp_path):
|
||||
result, recorder = await run(tmp_path, turns_with_one_revision())
|
||||
|
||||
assert result.status == "completed"
|
||||
record = recorder.lessons[result.execution_id]
|
||||
|
||||
assert of_type(record, "revision") == [
|
||||
{
|
||||
"type": "revision",
|
||||
"deliverable": "deployment_guidance",
|
||||
"reason": "Fix the ingress block.",
|
||||
"requested_by": "orchestrator",
|
||||
}
|
||||
]
|
||||
assert of_type(record, "failed_check") == [
|
||||
{"type": "failed_check", "attempt": 1, **FAILED}
|
||||
]
|
||||
assert of_type(record, "proposed_addition") == [
|
||||
{"type": "proposed_addition", "attempt": 1, **ADDITION}
|
||||
]
|
||||
refusals = of_type(record, "refusal")
|
||||
assert {(r["role"], r["tool"], r["deliverable"]) for r in refusals} == {
|
||||
("orchestrator", "delegate", None),
|
||||
(
|
||||
specialist(K.DEPLOYMENT_GUIDANCE),
|
||||
"submit_deliverable",
|
||||
"deployment_guidance",
|
||||
),
|
||||
}
|
||||
assert all(r["message"].startswith(("Refused", "Not submitted")) for r in refusals)
|
||||
|
||||
|
||||
async def test_the_record_names_the_execution_and_the_skills_loaded(tmp_path):
|
||||
result, recorder = await run(tmp_path, turns_with_one_revision())
|
||||
|
||||
record = recorder.lessons[result.execution_id]
|
||||
|
||||
assert record["execution_id"] == result.execution_id
|
||||
assert record["status"] == "completed"
|
||||
assert record["role_skills"]["orchestrator"] == ["solution-architecture"]
|
||||
assert record["skill_versions"] == {"solution-architecture": "abc123"}
|
||||
|
||||
|
||||
async def test_the_record_is_also_written_beside_the_deliverables_tree(tmp_path):
|
||||
result, recorder = await run(tmp_path, turns_with_one_revision())
|
||||
|
||||
path = (
|
||||
tmp_path
|
||||
/ "lessons"
|
||||
/ "candidates"
|
||||
/ "unassigned"
|
||||
/ f"{result.execution_id}.json"
|
||||
)
|
||||
|
||||
assert (
|
||||
json.loads(path.read_text(encoding="utf-8"))
|
||||
== (recorder.lessons[result.execution_id])
|
||||
)
|
||||
|
||||
|
||||
async def test_an_execution_that_ends_with_warnings_still_records_lessons(tmp_path):
|
||||
turns = {
|
||||
"orchestrator": [[delegate(K.REQUIREMENTS)], []],
|
||||
specialist(K.REQUIREMENTS): [[]], # stops without submitting
|
||||
}
|
||||
|
||||
result, recorder = await run(tmp_path, turns)
|
||||
|
||||
assert result.status == "completed_with_warnings"
|
||||
record = recorder.lessons[result.execution_id]
|
||||
assert of_type(record, "unfinished") == [
|
||||
{
|
||||
"type": "unfinished",
|
||||
"deliverable": "requirements",
|
||||
"role": specialist(K.REQUIREMENTS),
|
||||
"reason": "stopped",
|
||||
}
|
||||
]
|
||||
# Only the scope had been produced, so only the scope was written.
|
||||
written = {path.name for path in tmp_path.rglob("deliverables/*.md")}
|
||||
assert written == {"scope.md"}
|
||||
|
||||
|
||||
def test_a_cloud_read_refusal_keeps_only_its_leading_words():
|
||||
"""What a cloud-read tool says may describe the user's environment."""
|
||||
item = refusal(
|
||||
"requirements-analyst",
|
||||
"requirements",
|
||||
"compute__list_instances",
|
||||
"cloud_read",
|
||||
"Refused: project acme-prod-1234 does not have the API enabled.",
|
||||
)
|
||||
|
||||
assert item["tool"] == "compute__list_instances"
|
||||
assert item["message"] == "Refused"
|
||||
assert "acme-prod-1234" not in item["message"]
|
||||
|
||||
|
||||
def test_refusal_messages_are_bounded():
|
||||
item = refusal(
|
||||
"validator", "validation_results", "terraform_validate", "general", "x" * 900
|
||||
)
|
||||
|
||||
assert len(item["message"]) == 500
|
||||
Reference in New Issue
Block a user