Files
gcp_solution_architecture_a…/tests/execution/test_lessons.py
Jonathan Boniface 545097530b
Some checks failed
validation / verify (push) Failing after 10s
feat: scoping is the first Phase, completed in conversation (iteration 9 step 2)
Every Execution now begins by agreeing the scope, before requirements.

- A new Phase and Deliverable, produced by a scope analyst with two tools:
  scope_read shows what is settled and what is not; scope_set records fields
  with where each value came from (the request, the customer's answer, or an
  assumption with its reason). Code checks each value against the catalogue.
- With someone present the analyst asks for what is missing; with nobody
  present it assumes it, visibly. With no spec it drafts one from the request.
  A value the customer stated in the file is changed only by their own answer.
- A submission cannot leave an essential field unknown. Code writes the
  completed spec, the data every later Specialist reads, and the information
  register, from what was recorded.
- The spec's status follows the Execution: under review, approved, then
  locked as a Baseline. A Change request starts from the delivered scope.
- The requirements analyst derives from the agreed scope and no longer asks
  for what it settles.

About 80 tests assumed requirements came first; tests/scoped.py runs scoping
ahead of a test's own script, and tests about order and counts are updated.
Not run with real models.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-11 18:17:32 +01:00

242 lines
8.0 KiB
Python

"""Lesson candidates: what an Execution keeps about what went wrong in it."""
import json
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
from app.states.lessons import refusal
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
K = DeliverableKind
P = PhaseId
ROLES = {
"orchestrator": RoleSpec(
"orchestrator", "Coordinate.", budget=40, skills=("solution-architecture",)
),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
FAILED = {"check": "K-2", "evidence": "The identity is shared with another service."}
ADDITION = {
"statement": "The service sets a request timeout.",
"applies_to": "E-1",
"reason": "NFR-1 has no Check on timeouts",
}
def delegate(*kinds: DeliverableKind) -> Call:
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
def approve(phase: PhaseId) -> Call:
return Call("request_approval", {"phase": phase.value, "summary": "done"})
def specialist(kind: DeliverableKind) -> str:
return DEPENDENCY_GRAPH[kind].specialist
def failing_validation() -> Call:
call = sample_submission(K.VALIDATION_RESULTS)
verdicts = json.loads(call.args["attachments"]["verdicts.json"])
verdicts["checks"][1] = {**FAILED, "verdict": "fail"}
verdicts["proposed_additions"] = [ADDITION]
return Call(
call.tool,
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
)
def turns_with_one_revision() -> dict:
once = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
return {
**once,
"orchestrator": [
# The first call is refused: requirements are not done yet.
[delegate(K.TECHNICAL_DECOMPOSITION), delegate(K.REQUIREMENTS)],
[delegate(K.TECHNICAL_DECOMPOSITION)],
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
[delegate(K.DESIGN_RECOMMENDATIONS)],
[delegate(K.TARGET_ARCHITECTURE)],
[
approve(P.SOLUTION_ARCHITECTURE),
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
],
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[
Call(
"reopen",
{"kind": "deployment_guidance", "reason": "Fix the ingress block."},
),
delegate(K.DEPLOYMENT_GUIDANCE),
],
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
[approve(P.SOLUTION_VALIDATION), delegate(K.SOLUTION_GUIDE)],
[approve(P.PACKAGING), Call("finish")],
],
specialist(K.DEPLOYMENT_GUIDANCE): [
[
Call("submit_deliverable", {"content": "No headings yet."}),
sample_submission(K.DEPLOYMENT_GUIDANCE),
],
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
],
specialist(K.VALIDATION_RESULTS): [
[failing_validation()],
[sample_submission(K.VALIDATION_RESULTS)],
],
}
async def run(tmp_path, turns, **options):
recorder = InMemoryRecorder()
engine = ExecutionEngine(
runtime=FakeAgentRuntime(turns),
roles=ROLES,
recorder=recorder,
checkpointer=InMemorySaver(),
output_root=tmp_path,
skill_versions={"solution-architecture": "abc123"},
**options,
)
result = await engine.start("Ingestion", pre_approved=True)
return result, recorder
def of_type(record: dict, kind: str) -> list[dict]:
return [c for c in record["candidates"] if c["type"] == kind]
async def test_a_finished_execution_records_what_went_wrong(tmp_path):
result, recorder = await run(tmp_path, turns_with_one_revision())
assert result.status == "completed"
record = recorder.lessons[result.execution_id]
assert of_type(record, "revision") == [
{
"type": "revision",
"deliverable": "deployment_guidance",
"reason": "Fix the ingress block.",
"requested_by": "orchestrator",
}
]
assert of_type(record, "failed_check") == [
{"type": "failed_check", "attempt": 1, **FAILED}
]
assert of_type(record, "proposed_addition") == [
{"type": "proposed_addition", "attempt": 1, **ADDITION}
]
refusals = of_type(record, "refusal")
assert {(r["role"], r["tool"], r["deliverable"]) for r in refusals} == {
("orchestrator", "delegate", None),
(
specialist(K.DEPLOYMENT_GUIDANCE),
"submit_deliverable",
"deployment_guidance",
),
}
assert all(r["message"].startswith(("Refused", "Not submitted")) for r in refusals)
async def test_the_record_names_the_execution_and_the_skills_loaded(tmp_path):
result, recorder = await run(tmp_path, turns_with_one_revision())
record = recorder.lessons[result.execution_id]
assert record["execution_id"] == result.execution_id
assert record["status"] == "completed"
assert record["role_skills"]["orchestrator"] == ["solution-architecture"]
assert record["skill_versions"] == {"solution-architecture": "abc123"}
async def test_the_record_is_also_written_beside_the_deliverables_tree(tmp_path):
result, recorder = await run(tmp_path, turns_with_one_revision())
path = (
tmp_path
/ "lessons"
/ "candidates"
/ "unassigned"
/ f"{result.execution_id}.json"
)
assert (
json.loads(path.read_text(encoding="utf-8"))
== (recorder.lessons[result.execution_id])
)
async def test_an_execution_that_ends_with_warnings_still_records_lessons(tmp_path):
turns = {
"orchestrator": [[delegate(K.REQUIREMENTS)], []],
specialist(K.REQUIREMENTS): [[]], # stops without submitting
}
result, recorder = await run(tmp_path, turns)
assert result.status == "completed_with_warnings"
record = recorder.lessons[result.execution_id]
assert of_type(record, "unfinished") == [
{
"type": "unfinished",
"deliverable": "requirements",
"role": specialist(K.REQUIREMENTS),
"reason": "stopped",
}
]
# Only the scope had been produced, so only the scope was written.
written = {path.name for path in tmp_path.rglob("deliverables/*.md")}
assert written == {"scope.md"}
async def test_a_cloud_read_refusal_keeps_only_its_leading_words(tmp_path):
async def list_instances() -> ToolResult:
return ToolResult("Discovery gap: project acme-prod-1234 API not enabled.")
tool = ToolSpec(
"compute__list_instances",
"List instances.",
{"type": "object", "properties": {}},
list_instances,
tier="cloud_read",
)
def factory(role, deliverable, working):
return [tool] if deliverable == "requirements" else []
turns = {
"orchestrator": [[delegate(K.REQUIREMENTS)], []],
specialist(K.REQUIREMENTS): [
[Call("compute__list_instances"), sample_submission(K.REQUIREMENTS)]
],
}
result, recorder = await run(tmp_path, turns, tool_factory=factory, max_revisions=3)
await_record = recorder.lessons[result.execution_id]
# allow_cloud_read is False in a pre-approved Execution, so the gate refuses.
[gap] = of_type(await_record, "refusal")
assert gap["tool"] == "compute__list_instances"
assert "acme-prod-1234" not in gap["message"]
def test_refusal_messages_are_bounded():
item = refusal(
"validator", "validation_results", "terraform_validate", "general", "x" * 900
)
assert len(item["message"]) == 500