"""Lesson candidates: what an Execution keeps about what went wrong in it.""" import json from langgraph.checkpoint.memory import InMemorySaver from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission from app.adk.runtime import RoleSpec, ToolResult, ToolSpec from app.agent import ExecutionEngine from app.recorder import InMemoryRecorder from app.states.lessons import refusal from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811 from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811 K = DeliverableKind P = PhaseId ROLES = { "orchestrator": RoleSpec( "orchestrator", "Coordinate.", budget=40, skills=("solution-architecture",) ), **{ spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15) for spec in DEPENDENCY_GRAPH.values() }, } FAILED = {"check": "K-2", "evidence": "The identity is shared with another service."} ADDITION = { "statement": "The service sets a request timeout.", "applies_to": "E-1", "reason": "NFR-1 has no Check on timeouts", } def delegate(*kinds: DeliverableKind) -> Call: return Call("delegate", {"kinds": [kind.value for kind in kinds]}) def approve(phase: PhaseId) -> Call: return Call("request_approval", {"phase": phase.value, "summary": "done"}) def specialist(kind: DeliverableKind) -> str: return DEPENDENCY_GRAPH[kind].specialist def failing_validation() -> Call: call = sample_submission(K.VALIDATION_RESULTS) verdicts = json.loads(call.args["attachments"]["verdicts.json"]) verdicts["checks"][1] = {**FAILED, "verdict": "fail"} verdicts["proposed_additions"] = [ADDITION] return Call( call.tool, {**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}}, ) def turns_with_one_revision() -> dict: once = { spec.specialist: [[sample_submission(kind)]] for kind, spec in DEPENDENCY_GRAPH.items() } return { **once, "orchestrator": [ # The first call is refused: requirements are not done yet. [delegate(K.TECHNICAL_DECOMPOSITION), delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)], [delegate(K.DESIGN_RECOMMENDATIONS)], [delegate(K.TARGET_ARCHITECTURE)], [ approve(P.SOLUTION_ARCHITECTURE), delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE), ], [approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)], [ Call( "reopen", {"kind": "deployment_guidance", "reason": "Fix the ingress block."}, ), delegate(K.DEPLOYMENT_GUIDANCE), ], [approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)], [approve(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)], [approve(P.PACKAGING), Call("finish")], ], specialist(K.DEPLOYMENT_GUIDANCE): [ [ Call("submit_deliverable", {"content": "No headings yet."}), sample_submission(K.DEPLOYMENT_GUIDANCE), ], [sample_submission(K.DEPLOYMENT_GUIDANCE)], ], specialist(K.VALIDATION_RESULTS): [ [failing_validation()], [sample_submission(K.VALIDATION_RESULTS)], ], } async def run(tmp_path, turns, **options): recorder = InMemoryRecorder() engine = ExecutionEngine( runtime=FakeAgentRuntime(turns), roles=ROLES, recorder=recorder, checkpointer=InMemorySaver(), output_root=tmp_path, skill_versions={"solution-architecture": "abc123"}, **options, ) result = await engine.start("Ingestion", pre_approved=True) return result, recorder def of_type(record: dict, kind: str) -> list[dict]: return [c for c in record["candidates"] if c["type"] == kind] async def test_a_finished_execution_records_what_went_wrong(tmp_path): result, recorder = await run(tmp_path, turns_with_one_revision()) assert result.status == "completed" record = recorder.lessons[result.execution_id] assert of_type(record, "revision") == [ { "type": "revision", "deliverable": "deployment_guidance", "reason": "Fix the ingress block.", "requested_by": "orchestrator", } ] assert of_type(record, "failed_check") == [ {"type": "failed_check", "attempt": 1, **FAILED} ] assert of_type(record, "proposed_addition") == [ {"type": "proposed_addition", "attempt": 1, **ADDITION} ] refusals = of_type(record, "refusal") assert {(r["role"], r["tool"], r["deliverable"]) for r in refusals} == { ("orchestrator", "delegate", None), ( specialist(K.DEPLOYMENT_GUIDANCE), "submit_deliverable", "deployment_guidance", ), } assert all(r["message"].startswith(("Refused", "Not submitted")) for r in refusals) async def test_the_record_names_the_execution_and_the_skills_loaded(tmp_path): result, recorder = await run(tmp_path, turns_with_one_revision()) record = recorder.lessons[result.execution_id] assert record["execution_id"] == result.execution_id assert record["status"] == "completed" assert record["role_skills"]["orchestrator"] == ["solution-architecture"] assert record["skill_versions"] == {"solution-architecture": "abc123"} async def test_the_record_is_also_written_beside_the_deliverables_tree(tmp_path): result, recorder = await run(tmp_path, turns_with_one_revision()) path = ( tmp_path / "lessons" / "candidates" / "unassigned" / f"{result.execution_id}.json" ) assert ( json.loads(path.read_text(encoding="utf-8")) == (recorder.lessons[result.execution_id]) ) async def test_an_execution_that_ends_with_warnings_still_records_lessons(tmp_path): turns = { "orchestrator": [[delegate(K.REQUIREMENTS)], []], specialist(K.REQUIREMENTS): [[]], # stops without submitting } result, recorder = await run(tmp_path, turns) assert result.status == "completed_with_warnings" record = recorder.lessons[result.execution_id] assert of_type(record, "unfinished") == [ { "type": "unfinished", "deliverable": "requirements", "role": specialist(K.REQUIREMENTS), "reason": "stopped", } ] # Only the scope had been produced, so only the scope was written. written = {path.name for path in tmp_path.rglob("deliverables/*.md")} assert written == {"scope.md"} async def test_a_cloud_read_refusal_keeps_only_its_leading_words(tmp_path): async def list_instances() -> ToolResult: return ToolResult("Discovery gap: project acme-prod-1234 API not enabled.") tool = ToolSpec( "compute__list_instances", "List instances.", {"type": "object", "properties": {}}, list_instances, tier="cloud_read", ) def factory(role, deliverable, working): return [tool] if deliverable == "requirements" else [] turns = { "orchestrator": [[delegate(K.REQUIREMENTS)], []], specialist(K.REQUIREMENTS): [ [Call("compute__list_instances"), sample_submission(K.REQUIREMENTS)] ], } result, recorder = await run(tmp_path, turns, tool_factory=factory, max_revisions=3) await_record = recorder.lessons[result.execution_id] # allow_cloud_read is False in a pre-approved Execution, so the gate refuses. [gap] = of_type(await_record, "refusal") assert gap["tool"] == "compute__list_instances" assert "acme-prod-1234" not in gap["message"] def test_refusal_messages_are_bounded(): item = refusal( "validator", "validation_results", "terraform_validate", "general", "x" * 900 ) assert len(item["message"]) == 500