233 lines
7.6 KiB
Python
233 lines
7.6 KiB
Python
"""Lesson candidates: what an Execution keeps about what went wrong in it."""
|
|
|
|
import json
|
|
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
|
from execution_engine.adk.runtime import RoleSpec
|
|
from execution_engine.agent import ExecutionEngine
|
|
from execution_engine.recorder import InMemoryRecorder
|
|
from execution_engine.states.lessons import refusal
|
|
from tests.reference_workflow.workflow import (
|
|
DEPENDENCY_GRAPH,
|
|
WORKFLOW,
|
|
DeliverableKind,
|
|
PhaseId,
|
|
sample_submission,
|
|
)
|
|
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
|
|
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
|
|
|
|
K = DeliverableKind
|
|
P = PhaseId
|
|
|
|
ROLES = {
|
|
"orchestrator": RoleSpec(
|
|
"orchestrator", "Coordinate.", budget=40, skills=("solution-architecture",)
|
|
),
|
|
**{
|
|
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
FAILED = {"check": "K-2", "evidence": "The identity is shared with another service."}
|
|
ADDITION = {
|
|
"statement": "The service sets a request timeout.",
|
|
"applies_to": "E-1",
|
|
"reason": "NFR-1 has no Check on timeouts",
|
|
}
|
|
|
|
|
|
def delegate(*kinds: DeliverableKind) -> Call:
|
|
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
|
|
|
|
|
def approve(phase: PhaseId) -> Call:
|
|
return Call("request_approval", {"phase": phase.value, "summary": "done"})
|
|
|
|
|
|
def specialist(kind: DeliverableKind) -> str:
|
|
return DEPENDENCY_GRAPH[kind].specialist
|
|
|
|
|
|
def failing_validation() -> Call:
|
|
call = sample_submission(K.VALIDATION_RESULTS)
|
|
verdicts = json.loads(call.args["attachments"]["verdicts.json"])
|
|
verdicts["checks"][1] = {**FAILED, "verdict": "fail"}
|
|
verdicts["proposed_additions"] = [ADDITION]
|
|
return Call(
|
|
call.tool,
|
|
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
|
|
)
|
|
|
|
|
|
def turns_with_one_revision() -> dict:
|
|
once = {
|
|
spec.specialist: [[sample_submission(kind)]]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
return {
|
|
**once,
|
|
"orchestrator": [
|
|
# The first call is refused: requirements are not done yet.
|
|
[delegate(K.TECHNICAL_DECOMPOSITION), delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
|
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
|
[delegate(K.TARGET_ARCHITECTURE)],
|
|
[
|
|
approve(P.SOLUTION_ARCHITECTURE),
|
|
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
|
],
|
|
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
|
[
|
|
Call(
|
|
"reopen",
|
|
{"kind": "deployment_guidance", "reason": "Fix the ingress block."},
|
|
),
|
|
delegate(K.DEPLOYMENT_GUIDANCE),
|
|
],
|
|
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
|
[approve(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
|
|
[approve(P.PACKAGING), Call("finish")],
|
|
],
|
|
specialist(K.DEPLOYMENT_GUIDANCE): [
|
|
[
|
|
Call("submit_deliverable", {"content": "No headings yet."}),
|
|
sample_submission(K.DEPLOYMENT_GUIDANCE),
|
|
],
|
|
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
|
],
|
|
specialist(K.VALIDATION_RESULTS): [
|
|
[failing_validation()],
|
|
[sample_submission(K.VALIDATION_RESULTS)],
|
|
],
|
|
}
|
|
|
|
|
|
async def run(tmp_path, turns, **options):
|
|
recorder = InMemoryRecorder()
|
|
engine = ExecutionEngine(
|
|
workflow=WORKFLOW,
|
|
runtime=FakeAgentRuntime(turns),
|
|
roles=ROLES,
|
|
recorder=recorder,
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
skill_versions={"solution-architecture": "abc123"},
|
|
**options,
|
|
)
|
|
result = await engine.start("Ingestion", pre_approved=True)
|
|
return result, recorder
|
|
|
|
|
|
def of_type(record: dict, kind: str) -> list[dict]:
|
|
return [c for c in record["candidates"] if c["type"] == kind]
|
|
|
|
|
|
async def test_a_finished_execution_records_what_went_wrong(tmp_path):
|
|
result, recorder = await run(tmp_path, turns_with_one_revision())
|
|
|
|
assert result.status == "completed"
|
|
record = recorder.lessons[result.execution_id]
|
|
|
|
assert of_type(record, "revision") == [
|
|
{
|
|
"type": "revision",
|
|
"deliverable": "deployment_guidance",
|
|
"reason": "Fix the ingress block.",
|
|
"requested_by": "orchestrator",
|
|
}
|
|
]
|
|
assert of_type(record, "failed_check") == [
|
|
{"type": "failed_check", "attempt": 1, **FAILED}
|
|
]
|
|
assert of_type(record, "proposed_addition") == [
|
|
{"type": "proposed_addition", "attempt": 1, **ADDITION}
|
|
]
|
|
refusals = of_type(record, "refusal")
|
|
assert {(r["role"], r["tool"], r["deliverable"]) for r in refusals} == {
|
|
("orchestrator", "delegate", None),
|
|
(
|
|
specialist(K.DEPLOYMENT_GUIDANCE),
|
|
"submit_deliverable",
|
|
"deployment_guidance",
|
|
),
|
|
}
|
|
assert all(r["message"].startswith(("Refused", "Not submitted")) for r in refusals)
|
|
|
|
|
|
async def test_the_record_names_the_execution_and_the_skills_loaded(tmp_path):
|
|
result, recorder = await run(tmp_path, turns_with_one_revision())
|
|
|
|
record = recorder.lessons[result.execution_id]
|
|
|
|
assert record["execution_id"] == result.execution_id
|
|
assert record["status"] == "completed"
|
|
assert record["role_skills"]["orchestrator"] == ["solution-architecture"]
|
|
assert record["skill_versions"] == {"solution-architecture": "abc123"}
|
|
|
|
|
|
async def test_the_record_is_also_written_beside_the_deliverables_tree(tmp_path):
|
|
result, recorder = await run(tmp_path, turns_with_one_revision())
|
|
|
|
path = (
|
|
tmp_path
|
|
/ "lessons"
|
|
/ "candidates"
|
|
/ "unassigned"
|
|
/ f"{result.execution_id}.json"
|
|
)
|
|
|
|
assert (
|
|
json.loads(path.read_text(encoding="utf-8"))
|
|
== (recorder.lessons[result.execution_id])
|
|
)
|
|
|
|
|
|
async def test_an_execution_that_ends_with_warnings_still_records_lessons(tmp_path):
|
|
turns = {
|
|
"orchestrator": [[delegate(K.REQUIREMENTS)], []],
|
|
specialist(K.REQUIREMENTS): [[]], # stops without submitting
|
|
}
|
|
|
|
result, recorder = await run(tmp_path, turns)
|
|
|
|
assert result.status == "completed_with_warnings"
|
|
record = recorder.lessons[result.execution_id]
|
|
assert of_type(record, "unfinished") == [
|
|
{
|
|
"type": "unfinished",
|
|
"deliverable": "requirements",
|
|
"role": specialist(K.REQUIREMENTS),
|
|
"reason": "stopped",
|
|
}
|
|
]
|
|
# Only the scope had been produced, so only the scope was written.
|
|
written = {path.name for path in tmp_path.rglob("deliverables/*.md")}
|
|
assert written == {"scope.md"}
|
|
|
|
|
|
def test_a_cloud_read_refusal_keeps_only_its_leading_words():
|
|
"""What a cloud-read tool says may describe the user's environment."""
|
|
item = refusal(
|
|
"requirements-analyst",
|
|
"requirements",
|
|
"compute__list_instances",
|
|
"cloud_read",
|
|
"Refused: project acme-prod-1234 does not have the API enabled.",
|
|
)
|
|
|
|
assert item["tool"] == "compute__list_instances"
|
|
assert item["message"] == "Refused"
|
|
assert "acme-prod-1234" not in item["message"]
|
|
|
|
|
|
def test_refusal_messages_are_bounded():
|
|
item = refusal(
|
|
"validator", "validation_results", "terraform_validate", "general", "x" * 900
|
|
)
|
|
|
|
assert len(item["message"]) == 500
|