773 lines
26 KiB
Python
773 lines
26 KiB
Python
"""Change requests (ADR-0006): an Execution that starts from a Baseline.
|
|
|
|
A Baseline is a completed Execution of the same Customer case. A Change request is a
|
|
new Execution that begins with the Baseline's Deliverables; the Baseline itself is
|
|
never modified.
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
|
from execution_engine.adk.runtime import NoDecider, RoleSpec
|
|
from execution_engine.agent import ExecutionEngine
|
|
from execution_engine.baseline import BaselineError, load_baseline
|
|
from execution_engine.nodes.router import Router, Thresholds
|
|
from execution_engine.recorder import InMemoryRecorder
|
|
from tests.reference_workflow.workflow import (
|
|
DEPENDENCY_GRAPH,
|
|
SAMPLE_DATA,
|
|
WORKFLOW,
|
|
DeliverableKind,
|
|
sample_submission,
|
|
)
|
|
from tests.scoped import ScopedEngine as ExecutionEngine # noqa: F811
|
|
from tests.scoped import ScopedRuntime as FakeAgentRuntime # noqa: F811
|
|
|
|
K = DeliverableKind
|
|
CASE = "acme-webhooks"
|
|
ROLES = {
|
|
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
|
**{
|
|
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
|
|
|
|
def once() -> dict:
|
|
return {
|
|
spec.specialist: [[sample_submission(kind)]]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
|
|
|
|
def revised(**changes) -> Call:
|
|
"""The requirements of a Change request: the Baseline's, each marked, plus what
|
|
`changes` adds or alters by id."""
|
|
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
|
for item in requirements:
|
|
item["change"] = "unchanged"
|
|
if item["id"] in changes:
|
|
item.update(changes.pop(item["id"]))
|
|
requirements += [{"id": key, **value} for key, value in changes.items()]
|
|
call = sample_submission(K.REQUIREMENTS)
|
|
return Call(
|
|
call.tool,
|
|
{
|
|
**call.args,
|
|
"attachments": {"requirements.json": json.dumps(requirements)},
|
|
},
|
|
)
|
|
|
|
|
|
RETENTION = {
|
|
"NFR-1": {
|
|
"change": "changed",
|
|
"text": "99.99% availability.",
|
|
"reason": "The service is now customer-facing.",
|
|
}
|
|
}
|
|
|
|
|
|
def changed(**overrides) -> dict:
|
|
"""Scripted turns for a Change request that changes NFR-1 and nothing else."""
|
|
turns = once()
|
|
turns["requirements-analyst"] = [[revised(**json.loads(json.dumps(RETENTION)))]]
|
|
turns.update(overrides)
|
|
return turns
|
|
|
|
|
|
def engine_for(tmp_path: Path, turns: dict, **options) -> ExecutionEngine:
|
|
return ExecutionEngine(
|
|
workflow=WORKFLOW,
|
|
runtime=FakeAgentRuntime(turns),
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
**options,
|
|
)
|
|
|
|
|
|
async def delivered(tmp_path: Path, case: str = CASE, execution_id: str = "v1"):
|
|
"""A completed Execution on disk, to be a Baseline."""
|
|
result = await engine_for(tmp_path, once()).start(
|
|
"Ingest webhooks",
|
|
customer_case=case,
|
|
pre_approved=True,
|
|
execution_id=execution_id,
|
|
)
|
|
assert result.status == "completed"
|
|
return result
|
|
|
|
|
|
def digest(folder: Path) -> dict[str, str]:
|
|
return {
|
|
str(path.relative_to(folder)): hashlib.sha256(path.read_bytes()).hexdigest()
|
|
for path in sorted(folder.rglob("*"))
|
|
if path.is_file()
|
|
}
|
|
|
|
|
|
def history(tmp_path: Path, case: str = CASE) -> list[dict]:
|
|
path = tmp_path / "deliverables" / "cases" / case / "intent" / "history.json"
|
|
return json.loads(path.read_text())["intents"]
|
|
|
|
|
|
# --- step 2: start against a Baseline --------------------------------------------------
|
|
|
|
|
|
async def test_a_baseline_is_read_from_its_files(tmp_path):
|
|
await delivered(tmp_path)
|
|
|
|
baseline = load_baseline(tmp_path, CASE, "v1")
|
|
|
|
assert baseline["execution_id"] == "v1"
|
|
assert baseline["workflow_request"] == "Ingest webhooks"
|
|
assert set(baseline["deliverables"]) == {kind.value for kind in DEPENDENCY_GRAPH}
|
|
graph = baseline["deliverables"]["target_architecture"]["attachments"]
|
|
assert "target-architecture.json" in graph
|
|
assert baseline["deliverables"]["requirements"]["content"].startswith("#")
|
|
|
|
|
|
async def test_a_change_request_starts_with_its_baseline_and_is_recorded(tmp_path):
|
|
await delivered(tmp_path)
|
|
|
|
result = await engine_for(tmp_path, changed()).start(
|
|
"Retention goes from 30 days to 7 years.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
assert result.status == "completed"
|
|
assert result.state["baseline"]["execution_id"] == "v1"
|
|
first, second = history(tmp_path)
|
|
assert first["baseline"] is None
|
|
assert second["baseline"] == "v1" and second["execution_id"] == "v2"
|
|
case = tmp_path / "deliverables" / "cases" / CASE
|
|
manifest = json.loads((case / "executions" / "v2" / "execution.json").read_text())
|
|
assert manifest["baseline"] == "v1"
|
|
assert "- Changes: `v1`" in (case / "intent" / "002.md").read_text()
|
|
|
|
|
|
async def test_the_baseline_is_never_modified(tmp_path):
|
|
await delivered(tmp_path)
|
|
folder = tmp_path / "deliverables" / "cases" / CASE / "executions" / "v1"
|
|
before = digest(folder)
|
|
|
|
await engine_for(tmp_path, changed()).start(
|
|
"Add a second region.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
assert digest(folder) == before
|
|
|
|
|
|
async def test_a_change_request_may_itself_be_a_baseline(tmp_path):
|
|
await delivered(tmp_path)
|
|
for version, previous in (("v2", "v1"), ("v3", "v2")):
|
|
await engine_for(tmp_path, changed()).start(
|
|
"Another change.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id=version,
|
|
baseline=previous,
|
|
)
|
|
|
|
assert [item["baseline"] for item in history(tmp_path)] == [None, "v1", "v2"]
|
|
assert load_baseline(tmp_path, CASE, "v3")["baseline"] == "v2"
|
|
|
|
|
|
async def test_without_a_baseline_an_execution_starts_clean(tmp_path):
|
|
result = await delivered(tmp_path)
|
|
|
|
assert result.state["baseline"] is None
|
|
manifest = json.loads(
|
|
(
|
|
tmp_path / "deliverables/cases" / CASE / "executions/v1/execution.json"
|
|
).read_text()
|
|
)
|
|
assert manifest["baseline"] is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"case, baseline, message",
|
|
[
|
|
(CASE, "nope", "is not an Execution of Customer case acme-webhooks"),
|
|
("other-client", "v1", "belongs to another Customer case"),
|
|
(None, "v1", "needs a customer_case"),
|
|
(CASE, "../v1", "is not an Execution id"),
|
|
],
|
|
)
|
|
async def test_a_baseline_that_cannot_be_one_refuses_the_start(
|
|
tmp_path, case, baseline, message
|
|
):
|
|
await delivered(tmp_path)
|
|
engine = engine_for(tmp_path, once())
|
|
|
|
with pytest.raises(BaselineError, match=message):
|
|
await engine.start(
|
|
"A change.", customer_case=case, execution_id="v2", baseline=baseline
|
|
)
|
|
|
|
# Refused before anything ran or was recorded.
|
|
assert [item["execution_id"] for item in history(tmp_path)] == ["v1"]
|
|
assert not (tmp_path / "deliverables/cases/other-client").exists()
|
|
|
|
|
|
async def test_an_execution_that_did_not_complete_cannot_be_a_baseline(tmp_path):
|
|
from tests.execution.test_router import failing
|
|
|
|
turns = once()
|
|
turns["validator"] = [[failing("K-1")]]
|
|
engine = engine_for(tmp_path, turns, max_revisions=0)
|
|
unfinished = await engine.start(
|
|
"Ingest webhooks", customer_case=CASE, pre_approved=True, execution_id="v1"
|
|
)
|
|
assert unfinished.status == "completed_with_warnings"
|
|
|
|
with pytest.raises(BaselineError, match="did not complete"):
|
|
load_baseline(tmp_path, CASE, "v1")
|
|
|
|
|
|
# --- step 3: the requirements say what changed -------------------------------------------
|
|
|
|
|
|
def requirements_check(revised_call: Call, baseline=None) -> tuple[str, ...]:
|
|
from tests.reference_workflow.workflow import check_submission
|
|
|
|
baseline = baseline or {
|
|
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
|
}
|
|
return check_submission(
|
|
K.REQUIREMENTS,
|
|
revised_call.args["content"],
|
|
revised_call.args["attachments"],
|
|
baseline=baseline,
|
|
)
|
|
|
|
|
|
def test_a_revised_requirement_list_that_accounts_for_every_change_is_accepted():
|
|
call = revised(
|
|
**{
|
|
"NFR-1": RETENTION["NFR-1"],
|
|
"NFR-2": {
|
|
"kind": "non_functional",
|
|
"text": "A second region.",
|
|
"conditions": ["The service runs in two regions."],
|
|
"change": "added",
|
|
"reason": "Customers are now in two continents.",
|
|
},
|
|
}
|
|
)
|
|
|
|
assert requirements_check(call) == ()
|
|
|
|
|
|
def test_every_requirement_in_a_change_request_says_whether_it_changed():
|
|
call = sample_submission(K.REQUIREMENTS) # no `change` on anything
|
|
|
|
missing = requirements_check(call)
|
|
|
|
assert "requirements.json: FR-1 needs a `change`" in missing[0]
|
|
assert len(missing) == 2
|
|
|
|
|
|
def test_a_change_needs_its_reason_and_an_unchanged_requirement_its_text():
|
|
no_reason = revised(**{"NFR-1": {"change": "changed", "text": "99.99%."}})
|
|
quietly = revised(**{"NFR-1": {"text": "99.99% availability."}})
|
|
quietly_other = revised(
|
|
**{"NFR-1": RETENTION["NFR-1"], "FR-1": {"text": "Accept events and files."}}
|
|
)
|
|
|
|
assert requirements_check(no_reason) == (
|
|
"requirements.json: NFR-1 is changed and needs a `reason`",
|
|
)
|
|
assert (
|
|
"NFR-1 is marked unchanged, but its text differs"
|
|
in (requirements_check(quietly)[0])
|
|
)
|
|
assert (
|
|
"FR-1 is marked unchanged, but its text differs"
|
|
in (requirements_check(quietly_other)[0])
|
|
)
|
|
|
|
|
|
def test_a_baseline_requirement_cannot_silently_disappear_or_be_reused():
|
|
call = revised(**{"NFR-1": RETENTION["NFR-1"]})
|
|
requirements = json.loads(call.args["attachments"]["requirements.json"])
|
|
dropped = Call(
|
|
call.tool,
|
|
{
|
|
**call.args,
|
|
"attachments": {"requirements.json": json.dumps(requirements[1:])},
|
|
},
|
|
)
|
|
reused = revised(
|
|
**{
|
|
"NFR-1": {
|
|
"change": "added",
|
|
"text": "Something else entirely.",
|
|
"reason": "New.",
|
|
}
|
|
}
|
|
)
|
|
invented = revised(
|
|
**{
|
|
"NFR-1": RETENTION["NFR-1"],
|
|
"NFR-9": {
|
|
"kind": "non_functional",
|
|
"text": "x",
|
|
"conditions": ["x"],
|
|
"change": "changed",
|
|
"reason": "y",
|
|
},
|
|
}
|
|
)
|
|
|
|
assert "Baseline requirement FR-1 is missing" in requirements_check(dropped)[0]
|
|
assert (
|
|
"NFR-1 is marked added, but the Baseline has that id"
|
|
in (requirements_check(reused)[0])
|
|
)
|
|
assert (
|
|
"NFR-9 is marked changed, but the Baseline has no such requirement"
|
|
in (requirements_check(invented)[0])
|
|
)
|
|
|
|
|
|
def test_a_change_request_must_change_something():
|
|
assert (
|
|
"nothing is marked added, changed or removed"
|
|
in (requirements_check(revised())[0])
|
|
)
|
|
|
|
|
|
def test_a_removed_requirement_stays_on_record_and_asks_nothing_downstream():
|
|
from tests.reference_workflow.chain import read_list
|
|
|
|
call = revised(
|
|
**{"NFR-1": {"change": "removed", "reason": "No longer a requirement."}}
|
|
)
|
|
upstream = {"requirements": {"attachments": call.args["attachments"]}}
|
|
|
|
assert requirements_check(call) == ()
|
|
live = read_list(upstream, "requirements", "requirements.json")
|
|
assert [item["id"] for item in live] == ["FR-1"]
|
|
assert '"NFR-1"' in call.args["attachments"]["requirements.json"]
|
|
|
|
|
|
# --- step 4: later Specialists revise from the Baseline ---------------------------------
|
|
|
|
|
|
async def test_each_specialist_is_given_its_delivered_version_and_what_changed(
|
|
tmp_path,
|
|
):
|
|
await delivered(tmp_path)
|
|
runtime = FakeAgentRuntime(changed())
|
|
engine = ExecutionEngine(
|
|
workflow=WORKFLOW,
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
)
|
|
|
|
await engine.start(
|
|
"Availability goes up.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
by_role = {run.role: run for run in runtime.runs}
|
|
analyst = by_role["requirements-analyst"].task.context["baseline"]
|
|
assert analyst["execution_id"] == "v1"
|
|
assert analyst["original_request"] == "Ingest webhooks"
|
|
assert "requirements.json" in analyst["deliverable"]["attachments"]
|
|
assert analyst["changes"] is None # nothing has changed yet
|
|
|
|
architect = by_role["target-architect"].task.context
|
|
assert architect["baseline"]["changes"]["requirements"] == [
|
|
{
|
|
"id": "NFR-1",
|
|
"change": "changed",
|
|
"reason": "The service is now customer-facing.",
|
|
"text": "99.99% availability.",
|
|
}
|
|
]
|
|
# Its working copy already holds the delivered graph.
|
|
assert architect["baseline"]["deliverable"]["attachments"][
|
|
"target-architecture.json"
|
|
]
|
|
assert "graph_read" in by_role["target-architect"].tools
|
|
# The guide is assembled afresh: the Baseline's is not handed back.
|
|
assert by_role["packager"].task.context["baseline"]["deliverable"] is None
|
|
|
|
|
|
async def test_a_description_whose_design_did_not_change_shape_is_carried_over(
|
|
tmp_path,
|
|
):
|
|
await delivered(tmp_path)
|
|
runtime = FakeAgentRuntime(changed())
|
|
engine = ExecutionEngine(
|
|
workflow=WORKFLOW,
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
)
|
|
|
|
result = await engine.start(
|
|
"Availability goes up.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
# The same Elements and Relationships as the Baseline: the describer is not run.
|
|
assert "describer" not in {run.role for run in runtime.runs}
|
|
assert result.state["deliverables"]["architecture_description"]["content"]
|
|
|
|
|
|
async def test_a_code_authors_working_copy_starts_as_the_delivered_code(tmp_path):
|
|
await delivered(tmp_path)
|
|
seen: dict = {}
|
|
|
|
def author(task):
|
|
seen["working_copy"] = task.context["working_copy"]
|
|
return [sample_submission(K.DEPLOYMENT_GUIDANCE)]
|
|
|
|
engine = engine_for(tmp_path, changed(**{"iac-author": [author]}))
|
|
|
|
await engine.start(
|
|
"Availability goes up.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
# It is told nothing of a working copy on a first attempt; the tools hold it.
|
|
assert seen["working_copy"] is None
|
|
|
|
|
|
# --- what changed, worked out from the ids ------------------------------------------------
|
|
|
|
|
|
def versions(graph_change=None, requirement_change=None) -> tuple[dict, dict]:
|
|
"""A Baseline and a revised set of Deliverables, as data files."""
|
|
|
|
def deliverables() -> dict:
|
|
return {
|
|
kind: {"attachments": {name: SAMPLE_DATA[name]}}
|
|
for kind, name in (
|
|
("requirements", "requirements.json"),
|
|
("technical_decomposition", "components.json"),
|
|
("design_recommendations", "recommendations.json"),
|
|
("target_architecture", "target-architecture.json"),
|
|
)
|
|
}
|
|
|
|
before, after = deliverables(), deliverables()
|
|
change = requirement_change or RETENTION
|
|
after["requirements"]["attachments"] = revised(
|
|
**json.loads(json.dumps(change))
|
|
).args["attachments"]
|
|
if graph_change:
|
|
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
|
graph_change(graph)
|
|
after["target_architecture"]["attachments"] = {
|
|
"target-architecture.json": json.dumps(graph)
|
|
}
|
|
return before, after
|
|
|
|
|
|
def test_a_change_inside_the_reach_of_a_changed_requirement_is_requested():
|
|
from tests.reference_workflow.changes import change_summary
|
|
|
|
def tighten(graph):
|
|
graph["checks"][0]["statement"] = "The service runs at least three instances."
|
|
|
|
summary = change_summary(*versions(tighten))
|
|
|
|
assert [item["id"] for item in summary["requirements"]] == ["NFR-1"]
|
|
assert summary["checks"] == {"added": [], "changed": ["K-1"], "removed": []}
|
|
assert summary["elements"] == {"added": [], "changed": [], "removed": []}
|
|
assert summary["unrequested"] == {}
|
|
|
|
|
|
def test_a_change_no_changed_requirement_reaches_is_listed_as_unrequested():
|
|
from tests.reference_workflow.changes import change_summary
|
|
|
|
def wander(graph):
|
|
# K-2 comes from DR-1, which did not change, on E-2, which realises nothing
|
|
# the changed requirement touches. NFR-1 reaches K-1 and E-1 only.
|
|
graph["checks"][1]["statement"] = "The identity has no keys."
|
|
|
|
change = {
|
|
"NFR-1": {"change": "changed", "text": "99.99%.", "reason": "Customer-facing."}
|
|
}
|
|
before, after = versions(wander, change)
|
|
# NFR-1 is served by C-1, which E-1 realises; cut that link to isolate E-2.
|
|
components = [{"id": "C-1", "name": "Ingress", "requirements": ["FR-1"]}]
|
|
for version in (before, after):
|
|
version["technical_decomposition"]["attachments"] = {
|
|
"components.json": json.dumps(components)
|
|
}
|
|
|
|
summary = change_summary(before, after)
|
|
|
|
assert summary["checks"]["changed"] == ["K-2"]
|
|
assert summary["unrequested"] == {"checks": ["K-2"]}
|
|
|
|
|
|
def test_added_and_removed_items_are_found_by_id():
|
|
from tests.reference_workflow.changes import change_summary
|
|
|
|
def grow(graph):
|
|
graph["elements"].append(
|
|
{
|
|
"id": "E-3",
|
|
"name": "Second region service",
|
|
"component": "C-1",
|
|
"product": "Example Run",
|
|
"resource_type": "example_run_service",
|
|
}
|
|
)
|
|
graph["checks"] = [graph["checks"][0]] + [
|
|
{"id": "K-3", "applies_to": "E-3", "statement": "x", "source": "NFR-1"}
|
|
]
|
|
|
|
summary = change_summary(*versions(grow))
|
|
|
|
assert summary["elements"]["added"] == ["E-3"]
|
|
assert summary["checks"] == {"added": ["K-3"], "changed": [], "removed": ["K-2"]}
|
|
# K-2 came from DR-1 on E-2: nothing that changed reaches it.
|
|
assert summary["unrequested"] == {"checks": ["K-2"]}
|
|
|
|
|
|
# --- step 5: Approvals show the change --------------------------------------------------
|
|
|
|
|
|
async def test_an_approval_in_a_change_request_shows_what_that_phase_changes(
|
|
tmp_path,
|
|
):
|
|
from tests.reference_workflow.presentation import describe_prompt
|
|
|
|
await delivered(tmp_path)
|
|
engine = engine_for(tmp_path, changed())
|
|
|
|
first = await engine.start(
|
|
"Availability goes up.", customer_case=CASE, execution_id="v2", baseline="v1"
|
|
)
|
|
|
|
approval = first.prompt["approval"]
|
|
assert approval["phase"] == "requirements_discovery"
|
|
assert approval["baseline"] == "v1"
|
|
assert [item["id"] for item in approval["changes"]["requirements"]] == ["NFR-1"]
|
|
text = describe_prompt(first.prompt)
|
|
assert "This changes the architecture delivered in v1." in text
|
|
assert (
|
|
"- Requirement NFR-1 changed: 99.99% availability. (The service is now "
|
|
"customer-facing.)"
|
|
) in text
|
|
|
|
second = await engine.resume(
|
|
"v2", {"approval": {"phase": "requirements_discovery", "decision": "approve"}}
|
|
)
|
|
|
|
# The design did not change, and the Approval says so.
|
|
design = second.prompt["approval"]
|
|
assert design["phase"] == "solution_architecture" and design["changes"] == {}
|
|
assert "Nothing in this Phase differs from it." in describe_prompt(second.prompt)
|
|
|
|
|
|
async def test_an_ordinary_approval_says_nothing_of_changes(tmp_path):
|
|
first = await engine_for(tmp_path, once()).start("Ingest webhooks")
|
|
|
|
assert "changes" not in first.prompt["approval"]
|
|
|
|
|
|
# --- step 6: the change summary and the line of versions ----------------------------------
|
|
|
|
|
|
async def test_a_change_request_writes_changes_json_and_a_full_set_of_files(tmp_path):
|
|
from tests.reference_workflow.changes import change_summary
|
|
|
|
await delivered(tmp_path)
|
|
|
|
result = await engine_for(tmp_path, changed()).start(
|
|
"Availability goes up.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
case = tmp_path / "deliverables" / "cases" / CASE / "executions"
|
|
written = json.loads((case / "v2" / "changes.json").read_text())
|
|
assert [item["id"] for item in written["requirements"]] == ["NFR-1"]
|
|
assert written["unrequested"] == {}
|
|
# It matches a comparison of the two Executions' data files.
|
|
assert written == change_summary(
|
|
load_baseline(tmp_path, CASE, "v1")["deliverables"],
|
|
result.state["deliverables"],
|
|
)
|
|
# The Change request stands on its own: every Deliverable is written in full.
|
|
manifest = json.loads((case / "v2" / "execution.json").read_text())
|
|
assert manifest["changes"] == "changes.json"
|
|
assert set(manifest["deliverables"]) == {kind.value for kind in DEPENDENCY_GRAPH}
|
|
assert (case / "v2" / "solution-architecture-guide.md").is_file()
|
|
assert not (case / "v1" / "changes.json").exists()
|
|
|
|
|
|
# --- only the pillars a change touches are consulted again ------------------------------
|
|
|
|
|
|
async def test_a_change_request_starts_from_the_baselines_pillar_advice(tmp_path):
|
|
"""The design advisor is told which pillars the change affects, and has the
|
|
Baseline's advice for the rest."""
|
|
from execution_engine.adk.fake_runtime import FakeDecider
|
|
from execution_engine.adk.runtime import Answer
|
|
from tests.reference_workflow.pillars import ADVICE_KEY
|
|
|
|
await delivered(tmp_path)
|
|
# A Baseline whose design advisor consulted two pillars.
|
|
folder = tmp_path / "deliverables" / "cases" / CASE / "executions" / "v1"
|
|
advice = {"security": "Use a dedicated identity.", "reliability": "Two zones."}
|
|
(folder / "working.json").write_text(
|
|
json.dumps({"design_recommendations": {ADVICE_KEY: advice}})
|
|
)
|
|
seen: dict = {}
|
|
|
|
def advisor(task):
|
|
seen["pillars_to_consult"] = task.context["pillars_to_consult"]
|
|
return [sample_submission(K.DESIGN_RECOMMENDATIONS)]
|
|
|
|
decider = FakeDecider(
|
|
{"security": Answer(0.3, 0.8), "reliability": Answer(1.9, 0.9)}
|
|
)
|
|
runtime = FakeAgentRuntime(changed(**{"design-advisor": [advisor]}))
|
|
engine = ExecutionEngine(
|
|
workflow=WORKFLOW,
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
decider=decider,
|
|
)
|
|
|
|
result = await engine.start(
|
|
"Availability goes up.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
assert seen["pillars_to_consult"] == ["reliability"]
|
|
state, _, _ = decider.asked[0]
|
|
assert "Requirement NFR-1 changed: 99.99% availability." in state
|
|
# The advice is carried into the new Execution, and written for the next change.
|
|
assert result.state["working"]["design_recommendations"][ADVICE_KEY] == advice
|
|
assert json.loads((folder.parent / "v2" / "working.json").read_text()) == {
|
|
"design_recommendations": {ADVICE_KEY: advice}
|
|
}
|
|
|
|
|
|
async def test_a_first_consultation_is_given_no_hint(tmp_path):
|
|
seen: dict = {}
|
|
|
|
def advisor(task):
|
|
seen["pillars_to_consult"] = task.context["pillars_to_consult"]
|
|
return [sample_submission(K.DESIGN_RECOMMENDATIONS)]
|
|
|
|
from execution_engine.adk.fake_runtime import FakeDecider
|
|
|
|
decider = FakeDecider({})
|
|
turns = once()
|
|
turns["design-advisor"] = [advisor]
|
|
engine = ExecutionEngine(
|
|
workflow=WORKFLOW,
|
|
runtime=FakeAgentRuntime(turns),
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
decider=decider,
|
|
)
|
|
|
|
await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert seen["pillars_to_consult"] is None and decider.asked == []
|
|
|
|
|
|
# --- changed, or only reworded? ---------------------------------------------------------
|
|
|
|
|
|
async def test_an_item_that_was_only_reworded_is_listed_apart_from_real_changes():
|
|
from execution_engine.adk.fake_runtime import FakeDecider
|
|
from execution_engine.adk.runtime import Answer
|
|
from tests.reference_workflow.changes import change_summary
|
|
from tests.reference_workflow.meaning import settle_meaning
|
|
|
|
def revise(graph):
|
|
graph["checks"][0]["statement"] = "The service runs at least three instances."
|
|
graph["checks"][1]["statement"] = "This service alone uses the identity."
|
|
|
|
before, after = versions(revise)
|
|
summary = change_summary(before, after)
|
|
assert summary["checks"]["changed"] == ["K-1", "K-2"]
|
|
decider = FakeDecider({"K-1": Answer(0.95, 0.9), "K-2": Answer(0.1, 0.8)})
|
|
|
|
settled = await settle_meaning(decider, summary, before, after, scope="v2")
|
|
|
|
assert settled["checks"]["changed"] == ["K-1"]
|
|
assert settled["checks"]["reworded"] == ["K-2"]
|
|
assert settled["meaning"] == {"K-1": 0.95, "K-2": 0.1}
|
|
# K-2 was outside the reach of what changed; reworded, it is no longer unrequested.
|
|
assert settled["unrequested"] == {}
|
|
_, questions, scope = decider.asked[0]
|
|
assert scope == "v2"
|
|
assert "The identity is used by this service only." in questions["K-2"].instructions
|
|
assert "This service alone uses the identity." in questions["K-2"].instructions
|
|
# The summary given in is not altered.
|
|
assert summary["checks"]["changed"] == ["K-1", "K-2"]
|
|
|
|
|
|
async def test_without_a_decision_everything_that_differs_stays_changed():
|
|
from tests.reference_workflow.changes import change_summary
|
|
from tests.reference_workflow.meaning import settle_meaning
|
|
|
|
def revise(graph):
|
|
graph["checks"][1]["statement"] = "This service alone uses the identity."
|
|
|
|
before, after = versions(revise)
|
|
summary = change_summary(before, after)
|
|
|
|
settled = await settle_meaning(NoDecider(), summary, before, after)
|
|
|
|
assert settled["checks"]["changed"] == ["K-2"]
|
|
assert "reworded" not in settled["checks"] and "meaning" not in settled
|