"""An Execution started from a scope spec (ADR-0008).""" import json import pytest import yaml from langgraph.checkpoint.memory import InMemorySaver from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission from app.adk.runtime import NoDecider, RoleSpec from app.agent import ExecutionEngine from app.nodes.router import Router, Thresholds from app.providers import load_provider from app.recorder import InMemoryRecorder from app.workflows.deliverables import DEPENDENCY_GRAPH from app.workflows.deliverables import DeliverableKind as K from tests.scoped import carrying ROLES = { "orchestrator": RoleSpec("orchestrator", "Unused.", budget=40), **{ spec.specialist: RoleSpec( spec.specialist, "Produce.", budget=15, skills=("base",) ) for spec in DEPENDENCY_GRAPH.values() }, } ANALYST = DEPENDENCY_GRAPH[K.REQUIREMENTS].specialist SCOPE_ANALYST = DEPENDENCY_GRAPH[K.SCOPE].specialist SPEC = """ metadata: {engagement_id: ENG-77} engagement: {kind: platform_foundation} intent: statement: Build us a landing zone. exclusions: [application workloads] estate: cloud_provider: gcp mode: greenfield regions: {primary: europe-west2} data_residency: United Kingdom compliance: frameworks: [DORA] data_classification: confidential networking: {topology: shared_vpc_hub_and_spoke, cidr_allocations: decide} run: {pre_approved: true, terraform_modules: false} """ STATED_CHOICES = [ "estate.regions.primary", "estate.data_residency", "compliance.frameworks", "compliance.data_classification", "networking.topology", ] class Recording(FakeAgentRuntime): def __init__(self, turns): super().__init__(turns) self.seen: dict[str, dict] = {} async def run(self, role, task, tools): self.seen[role.name] = { "skills": role.skills, "scope": task.context.get("scope"), } return await super().run(role, task, tools) def engine(tmp_path) -> tuple[ExecutionEngine, Recording]: bundle = load_provider("gcp") runtime = Recording( { **{ spec.specialist: [[sample_submission(kind)]] for kind, spec in DEPENDENCY_GRAPH.items() }, # The requirements carry every choice SPEC states, and a Check holds # the design to them. **carrying(STATED_CHOICES), } ) built = ExecutionEngine( runtime=runtime, roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), scope_catalogue=bundle.scope_catalogue(), scope_values=bundle.scope_values, scope_skill=bundle.scope_skill, ) return built, runtime async def test_a_spec_alone_starts_an_execution_and_names_its_customer_case(tmp_path): built, _ = engine(tmp_path) result = await built.start("", scope=SPEC, execution_id="v1") assert result.status == "completed" state = result.state # The spec said pre-approved, named the case, and gave the request. assert state["pre_approved"] is True and state["customer_case"] == "eng-77" assert state["workflow_request"] == "Build us a landing zone." assert state["scope"]["status"] == "APPROVED" async def test_the_supplied_file_is_kept_unaltered_beside_a_completed_copy(tmp_path): built, _ = engine(tmp_path) await built.start("", scope=SPEC, execution_id="v1") intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent" assert (intent / "001.scope.yaml").read_text() == SPEC completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text()) # Its Execution completed, so this copy is the record a change starts from. assert completed["metadata"]["status"] == "BASELINE_LOCKED" assert completed["networking"]["cidr_allocations"] == "decide" assert completed["provenance"]["estate.regions.primary"] == "supplied" assert [a["field"] for a in completed["inferred_assumptions"]] == [ "networking.cidr_allocations" ] history = json.loads((intent / "history.json").read_text()) assert history["intents"][0]["scope"] == { "completed": "intent/001.scope.completed.yaml", "supplied": "intent/001.scope.yaml", } manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text()) assert manifest["scope"]["status"] == "BASELINE_LOCKED" assert manifest["scope"]["readiness"]["assumed"] == 1 async def test_every_specialist_is_shown_the_spec_and_given_the_skill(tmp_path): built, runtime = engine(tmp_path) await built.start("", scope=SPEC, execution_id="v1") for name, seen in runtime.seen.items(): assert seen["skills"] == ("base", "scoping"), name assert seen["scope"]["kind"] == "platform_foundation" stated = runtime.seen[ANALYST]["scope"]["stated"] assert stated["networking"]["topology"] == "shared_vpc_hub_and_spoke" assert stated["intent"]["exclusions"] == ["application workloads"] assert runtime.seen[ANALYST]["scope"]["to_decide"][0]["field"] == ( "networking.cidr_allocations" ) def settle(**values) -> Call: """What a scope analyst records when it must assume what nobody stated.""" return Call( "scope_set", { "entries": [ { "field": field, "value": value, "source": "assumed", "note": "A default.", } for field, value in values.items() ] }, ) DRAFTED = [ Call( "scope_set", { "entries": [ { "field": "intent.statement", "value": "Ingest webhooks", "source": "request", }, {"field": "engagement.kind", "value": "workload", "source": "request"}, ] }, ), settle( **{ "estate.cloud_provider": "gcp", "estate.mode": "greenfield", "estate.regions.primary": "europe-west2", "estate.data_residency": "United Kingdom", "workload.users.who": "systems", "workload.users.count": 10, "workload.availability.target": "99.9%", "compliance.frameworks": [], "compliance.data_classification": "internal", "cost.monthly_budget": 500, } ), ] async def test_without_a_spec_the_scope_analyst_drafts_one_from_the_request(tmp_path): built, runtime = engine(tmp_path) runtime._turns[SCOPE_ANALYST].clear() runtime._turns[SCOPE_ANALYST].append([*DRAFTED, sample_submission(K.SCOPE)]) result = await built.start( "Ingest webhooks", customer_case="acme", pre_approved=True, execution_id="v1" ) assert result.status == "completed" intent = tmp_path / "deliverables" / "cases" / "acme" / "intent" # Nothing was supplied, so there is no supplied file; the completed copy is # what the analyst drafted. assert not (intent / "001.scope.yaml").exists() completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text()) assert completed["intent"]["statement"] == "Ingest webhooks" assert completed["metadata"]["customer_case"] == "acme" assert completed["provenance"]["intent.statement"] == "request" assert completed["provenance"]["estate.regions.primary"] == "assumed" assert completed["readiness"]["assumed"] == 10 assumed = {a["field"] for a in completed["inferred_assumptions"]} assert "cost.monthly_budget" in assumed # What was assumed is shown apart from what the customer stated. scope = runtime.seen[ANALYST]["scope"] assert scope["stated"]["intent"]["statement"] == "Ingest webhooks" assert "estate" not in scope["stated"] assert {"field": "cost.monthly_budget", "value": 500, "why": "A default."} in ( scope["assumed"] ) document = tmp_path / "deliverables" / "cases" / "acme" / "executions" / "v1" register = (document / "deliverables" / "scope.md").read_text() assert "## Information register" in register and "### Assumptions" in register assert "| intent.statement | Ingest webhooks | the request |" in register async def test_a_scope_with_an_essential_field_unsettled_cannot_be_submitted(tmp_path): built, runtime = engine(tmp_path) runtime._turns[SCOPE_ANALYST].clear() # It records the statement and nothing else, then tries to submit, twice. runtime._turns[SCOPE_ANALYST].append( [DRAFTED[0], sample_submission(K.SCOPE), sample_submission(K.SCOPE)] ) result = await built.start("Ingest webhooks", pre_approved=True) run = next(r for r in runtime.runs if r.role == SCOPE_ANALYST) refused = run.results[1][1] assert refused.startswith("Not submitted. Essential fields of the scope spec") assert "estate.regions.primary" in refused assert result.status == "completed_with_warnings" async def test_what_the_customer_stated_is_not_overwritten_by_the_analyst(tmp_path): built, runtime = engine(tmp_path) runtime._turns[SCOPE_ANALYST].clear() runtime._turns[SCOPE_ANALYST].append( [ Call("scope_read"), settle(**{"estate.regions.primary": "us-central1", "nonsense.field": 1}), Call( "scope_set", { "entries": [ { "field": "estate.regions.primary", "value": "europe-west1", "source": "answered", } ] }, ), Call( "scope_set", { "entries": [ { "field": "cost.monthly_budget", "value": "lots", "source": "answered", } ] }, ), sample_submission(K.SCOPE), ] ) await built.start("", scope=SPEC, execution_id="v1") run = next(r for r in runtime.runs if r.role == SCOPE_ANALYST) read, assumed, answered, mistyped = (reply for _, reply in run.results[:4]) assert "Kind of engagement: platform_foundation" in read assert "Left to you to decide: networking.cidr_allocations" in read assert "estate.regions.primary was stated in the scope spec" in assumed assert "nonsense.field is not a field" in assumed # The customer's own answer does change what the file said. assert answered == "Recorded 1: estate.regions.primary." assert "cost.monthly_budget: must be a number" in mistyped intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent" completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text()) assert completed["estate"]["regions"]["primary"] == "europe-west1" assert completed["provenance"]["estate.regions.primary"] == "answered" async def test_the_scope_is_approved_before_requirements_begin(tmp_path): built, runtime = engine(tmp_path) waiting = await built.start("", scope=SPEC, pre_approved=False, execution_id="v1") # Only the scope analyst has run, and the customer is asked to approve the scope. assert {run.role for run in runtime.runs} == {SCOPE_ANALYST} assert waiting.prompt["approval"]["phase"] == "scoping" intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent" completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text()) assert completed["metadata"]["status"] == "UNDER_REVIEW" approved = await built.resume( "v1", {"approval": {"phase": "scoping", "decision": "approve"}} ) assert ANALYST in {run.role for run in runtime.runs} assert approved.prompt["approval"]["phase"] == "requirements_discovery" completed = yaml.safe_load((intent / "001.scope.completed.yaml").read_text()) assert completed["metadata"]["status"] == "APPROVED" async def test_an_essential_gap_refuses_a_start_with_nobody_to_ask(tmp_path): built, runtime = engine(tmp_path) gappy = SPEC.replace(" data_residency: United Kingdom\n", "") with pytest.raises(ValueError, match="estate.data_residency") as refused: await built.start("", scope=gappy) assert "nobody to ask" in str(refused.value) assert runtime.seen == {} and not list(tmp_path.rglob("history.json")) # With someone present the same spec starts, and the gap is theirs to answer. waiting = await built.start("", scope=gappy, pre_approved=False, execution_id="v2") assert waiting.state["scope"]["status"] == "DRAFT" asked = waiting.state["scope"]["context"]["to_ask"] assert asked[0] == { "field": "estate.data_residency", "prompt": "Where must data stay?", "severity": "HIGH", } async def test_a_spec_that_cannot_be_read_refuses_the_start(tmp_path): built, runtime = engine(tmp_path) with pytest.raises(ValueError, match="networking.topolgy is not a field"): await built.start("", scope=SPEC.replace("topology", "topolgy")) with pytest.raises(ValueError, match="must be one of"): await built.start( "", scope=SPEC.replace("cloud_provider: gcp", "cloud_provider: aws") ) with pytest.raises(ValueError, match="started for another-case"): await built.start("", scope=SPEC, customer_case="another-case") assert runtime.seen == {} async def test_what_the_caller_says_outright_wins_over_the_specs_run_options(tmp_path): built, _ = engine(tmp_path) waiting = await built.start("", scope=SPEC, pre_approved=False, execution_id="v1") assert waiting.state["pre_approved"] is False assert waiting.status == "awaiting_user" async def test_a_request_is_still_needed_when_the_spec_has_no_statement(tmp_path): built, _ = engine(tmp_path) with pytest.raises(ValueError, match="A request is needed"): await built.start( "", scope=SPEC.replace(" statement: Build us a landing zone.\n", ""), pre_approved=False, ) async def test_a_change_request_starts_from_the_scope_that_was_delivered(tmp_path): built, runtime = engine(tmp_path) await built.start("", scope=SPEC, execution_id="v1") for name, turns in runtime._turns.items(): kind = next( k for k, spec in DEPENDENCY_GRAPH.items() if spec.specialist == name ) turns.append([sample_submission(kind)]) runtime._turns[SCOPE_ANALYST].clear() runtime._turns[SCOPE_ANALYST].append( [ Call("scope_read"), Call( "scope_set", { "entries": [ { "field": "estate.regions.secondary", "value": "europe-west1", "source": "answered", } ] }, ), sample_submission(K.SCOPE), ] ) await built.start( "We now need a second region.", customer_case="eng-77", pre_approved=True, execution_id="v2", baseline="v1", ) run = [r for r in runtime.runs if r.role == SCOPE_ANALYST][-1] # The delivered spec was there to read before anything was recorded. assert "Kind of engagement: platform_foundation" in run.results[0][1] intent = tmp_path / "deliverables" / "cases" / "eng-77" / "intent" changed = yaml.safe_load((intent / "002.scope.completed.yaml").read_text()) assert changed["estate"]["regions"] == { "primary": "europe-west2", "secondary": "europe-west1", "resiliency_tier": "ask", } assert changed["provenance"]["estate.regions.primary"] == "supplied" assert changed["provenance"]["estate.regions.secondary"] == "answered" # The Baseline's own copy is untouched. first = yaml.safe_load((intent / "001.scope.completed.yaml").read_text()) assert first["estate"]["regions"]["secondary"] == "ask"