Iteration 4: - Overlay Skills (step 12): an Execution names an organisation's standards Skill from the folders the deployment configures; recorded with its version. An example standards Skill and an evaluation case that uses it. - Lessons (step 10): a prove command that compares a run with lessons against one without, and triage of evidence an existing lesson already covers. Iteration 5: - Step 8: the references of a Specialist's Skills its task needs are named in its context as read_first. - Step 14: a remote MCP server's reply that instructs the agent is withheld; the user's own text is flagged as a risk and nothing more. - Step 15: in a Change request, items only reworded are listed apart from real changes. - Step 19: each Execution's design recommendations are judged when it ends and the scores kept in its record. The evaluation takes its rubric from the same place. Each question was calibrated against live Jev; none has run in a case. Steps 6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
765 lines
26 KiB
Python
765 lines
26 KiB
Python
"""Change requests (ADR-0006): an Execution that starts from a Baseline.
|
|
|
|
A Baseline is a completed Execution of the same Customer case. A Change request is a
|
|
new Execution that begins with the Baseline's Deliverables; the Baseline itself is
|
|
never modified.
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from app.adk.fake_runtime import (
|
|
SAMPLE_DATA,
|
|
Call,
|
|
FakeAgentRuntime,
|
|
sample_submission,
|
|
)
|
|
from app.adk.runtime import NoDecider, RoleSpec
|
|
from app.agent import ExecutionEngine
|
|
from app.baseline import BaselineError, load_baseline
|
|
from app.nodes.router import Router, Thresholds
|
|
from app.recorder import InMemoryRecorder
|
|
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
|
|
|
K = DeliverableKind
|
|
CASE = "acme-webhooks"
|
|
ROLES = {
|
|
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
|
**{
|
|
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
|
|
|
|
def once() -> dict:
|
|
return {
|
|
spec.specialist: [[sample_submission(kind)]]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
|
|
|
|
def revised(**changes) -> Call:
|
|
"""The requirements of a Change request: the Baseline's, each marked, plus what
|
|
`changes` adds or alters by id."""
|
|
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
|
for item in requirements:
|
|
item["change"] = "unchanged"
|
|
if item["id"] in changes:
|
|
item.update(changes.pop(item["id"]))
|
|
requirements += [{"id": key, **value} for key, value in changes.items()]
|
|
call = sample_submission(K.REQUIREMENTS)
|
|
return Call(
|
|
call.tool,
|
|
{
|
|
**call.args,
|
|
"attachments": {"requirements.json": json.dumps(requirements)},
|
|
},
|
|
)
|
|
|
|
|
|
RETENTION = {
|
|
"NFR-1": {
|
|
"change": "changed",
|
|
"text": "99.99% availability.",
|
|
"reason": "The service is now customer-facing.",
|
|
}
|
|
}
|
|
|
|
|
|
def changed(**overrides) -> dict:
|
|
"""Scripted turns for a Change request that changes NFR-1 and nothing else."""
|
|
turns = once()
|
|
turns["requirements-analyst"] = [[revised(**json.loads(json.dumps(RETENTION)))]]
|
|
turns.update(overrides)
|
|
return turns
|
|
|
|
|
|
def engine_for(tmp_path: Path, turns: dict, **options) -> ExecutionEngine:
|
|
return ExecutionEngine(
|
|
runtime=FakeAgentRuntime(turns),
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
**options,
|
|
)
|
|
|
|
|
|
async def delivered(tmp_path: Path, case: str = CASE, execution_id: str = "v1"):
|
|
"""A completed Execution on disk, to be a Baseline."""
|
|
result = await engine_for(tmp_path, once()).start(
|
|
"Ingest webhooks",
|
|
customer_case=case,
|
|
pre_approved=True,
|
|
execution_id=execution_id,
|
|
)
|
|
assert result.status == "completed"
|
|
return result
|
|
|
|
|
|
def digest(folder: Path) -> dict[str, str]:
|
|
return {
|
|
str(path.relative_to(folder)): hashlib.sha256(path.read_bytes()).hexdigest()
|
|
for path in sorted(folder.rglob("*"))
|
|
if path.is_file()
|
|
}
|
|
|
|
|
|
def history(tmp_path: Path, case: str = CASE) -> list[dict]:
|
|
path = tmp_path / "deliverables" / "cases" / case / "intent" / "history.json"
|
|
return json.loads(path.read_text())["intents"]
|
|
|
|
|
|
# --- step 2: start against a Baseline --------------------------------------------------
|
|
|
|
|
|
async def test_a_baseline_is_read_from_its_files(tmp_path):
|
|
await delivered(tmp_path)
|
|
|
|
baseline = load_baseline(tmp_path, CASE, "v1")
|
|
|
|
assert baseline["execution_id"] == "v1"
|
|
assert baseline["workflow_request"] == "Ingest webhooks"
|
|
assert set(baseline["deliverables"]) == {kind.value for kind in DEPENDENCY_GRAPH}
|
|
graph = baseline["deliverables"]["target_architecture"]["attachments"]
|
|
assert "target-architecture.json" in graph
|
|
assert baseline["deliverables"]["requirements"]["content"].startswith("#")
|
|
|
|
|
|
async def test_a_change_request_starts_with_its_baseline_and_is_recorded(tmp_path):
|
|
await delivered(tmp_path)
|
|
|
|
result = await engine_for(tmp_path, changed()).start(
|
|
"Retention goes from 30 days to 7 years.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
assert result.status == "completed"
|
|
assert result.state["baseline"]["execution_id"] == "v1"
|
|
first, second = history(tmp_path)
|
|
assert first["baseline"] is None
|
|
assert second["baseline"] == "v1" and second["execution_id"] == "v2"
|
|
case = tmp_path / "deliverables" / "cases" / CASE
|
|
manifest = json.loads((case / "executions" / "v2" / "execution.json").read_text())
|
|
assert manifest["baseline"] == "v1"
|
|
assert "- Changes: `v1`" in (case / "intent" / "002.md").read_text()
|
|
|
|
|
|
async def test_the_baseline_is_never_modified(tmp_path):
|
|
await delivered(tmp_path)
|
|
folder = tmp_path / "deliverables" / "cases" / CASE / "executions" / "v1"
|
|
before = digest(folder)
|
|
|
|
await engine_for(tmp_path, changed()).start(
|
|
"Add a second region.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
assert digest(folder) == before
|
|
|
|
|
|
async def test_a_change_request_may_itself_be_a_baseline(tmp_path):
|
|
await delivered(tmp_path)
|
|
for version, previous in (("v2", "v1"), ("v3", "v2")):
|
|
await engine_for(tmp_path, changed()).start(
|
|
"Another change.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id=version,
|
|
baseline=previous,
|
|
)
|
|
|
|
assert [item["baseline"] for item in history(tmp_path)] == [None, "v1", "v2"]
|
|
assert load_baseline(tmp_path, CASE, "v3")["baseline"] == "v2"
|
|
|
|
|
|
async def test_without_a_baseline_an_execution_starts_clean(tmp_path):
|
|
result = await delivered(tmp_path)
|
|
|
|
assert result.state["baseline"] is None
|
|
manifest = json.loads(
|
|
(
|
|
tmp_path / "deliverables/cases" / CASE / "executions/v1/execution.json"
|
|
).read_text()
|
|
)
|
|
assert manifest["baseline"] is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"case, baseline, message",
|
|
[
|
|
(CASE, "nope", "is not an Execution of Customer case acme-webhooks"),
|
|
("other-client", "v1", "belongs to another Customer case"),
|
|
(None, "v1", "needs a customer_case"),
|
|
(CASE, "../v1", "is not an Execution id"),
|
|
],
|
|
)
|
|
async def test_a_baseline_that_cannot_be_one_refuses_the_start(
|
|
tmp_path, case, baseline, message
|
|
):
|
|
await delivered(tmp_path)
|
|
engine = engine_for(tmp_path, once())
|
|
|
|
with pytest.raises(BaselineError, match=message):
|
|
await engine.start(
|
|
"A change.", customer_case=case, execution_id="v2", baseline=baseline
|
|
)
|
|
|
|
# Refused before anything ran or was recorded.
|
|
assert [item["execution_id"] for item in history(tmp_path)] == ["v1"]
|
|
assert not (tmp_path / "deliverables/cases/other-client").exists()
|
|
|
|
|
|
async def test_an_execution_that_did_not_complete_cannot_be_a_baseline(tmp_path):
|
|
from tests.execution.test_router import failing
|
|
|
|
turns = once()
|
|
turns["validator"] = [[failing("K-1")]]
|
|
engine = engine_for(tmp_path, turns, max_revisions=0)
|
|
unfinished = await engine.start(
|
|
"Ingest webhooks", customer_case=CASE, pre_approved=True, execution_id="v1"
|
|
)
|
|
assert unfinished.status == "completed_with_warnings"
|
|
|
|
with pytest.raises(BaselineError, match="did not complete"):
|
|
load_baseline(tmp_path, CASE, "v1")
|
|
|
|
|
|
# --- step 3: the requirements say what changed -------------------------------------------
|
|
|
|
|
|
def requirements_check(revised_call: Call, baseline=None) -> tuple[str, ...]:
|
|
from app.workflows.deliverables import check_submission
|
|
|
|
baseline = baseline or {
|
|
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
|
}
|
|
return check_submission(
|
|
K.REQUIREMENTS,
|
|
revised_call.args["content"],
|
|
revised_call.args["attachments"],
|
|
baseline=baseline,
|
|
)
|
|
|
|
|
|
def test_a_revised_requirement_list_that_accounts_for_every_change_is_accepted():
|
|
call = revised(
|
|
**{
|
|
"NFR-1": RETENTION["NFR-1"],
|
|
"NFR-2": {
|
|
"kind": "non_functional",
|
|
"text": "A second region.",
|
|
"conditions": ["The service runs in two regions."],
|
|
"change": "added",
|
|
"reason": "Customers are now in two continents.",
|
|
},
|
|
}
|
|
)
|
|
|
|
assert requirements_check(call) == ()
|
|
|
|
|
|
def test_every_requirement_in_a_change_request_says_whether_it_changed():
|
|
call = sample_submission(K.REQUIREMENTS) # no `change` on anything
|
|
|
|
missing = requirements_check(call)
|
|
|
|
assert "requirements.json: FR-1 needs a `change`" in missing[0]
|
|
assert len(missing) == 2
|
|
|
|
|
|
def test_a_change_needs_its_reason_and_an_unchanged_requirement_its_text():
|
|
no_reason = revised(**{"NFR-1": {"change": "changed", "text": "99.99%."}})
|
|
quietly = revised(**{"NFR-1": {"text": "99.99% availability."}})
|
|
quietly_other = revised(
|
|
**{"NFR-1": RETENTION["NFR-1"], "FR-1": {"text": "Accept events and files."}}
|
|
)
|
|
|
|
assert requirements_check(no_reason) == (
|
|
"requirements.json: NFR-1 is changed and needs a `reason`",
|
|
)
|
|
assert (
|
|
"NFR-1 is marked unchanged, but its text differs"
|
|
in (requirements_check(quietly)[0])
|
|
)
|
|
assert (
|
|
"FR-1 is marked unchanged, but its text differs"
|
|
in (requirements_check(quietly_other)[0])
|
|
)
|
|
|
|
|
|
def test_a_baseline_requirement_cannot_silently_disappear_or_be_reused():
|
|
call = revised(**{"NFR-1": RETENTION["NFR-1"]})
|
|
requirements = json.loads(call.args["attachments"]["requirements.json"])
|
|
dropped = Call(
|
|
call.tool,
|
|
{
|
|
**call.args,
|
|
"attachments": {"requirements.json": json.dumps(requirements[1:])},
|
|
},
|
|
)
|
|
reused = revised(
|
|
**{
|
|
"NFR-1": {
|
|
"change": "added",
|
|
"text": "Something else entirely.",
|
|
"reason": "New.",
|
|
}
|
|
}
|
|
)
|
|
invented = revised(
|
|
**{
|
|
"NFR-1": RETENTION["NFR-1"],
|
|
"NFR-9": {
|
|
"kind": "non_functional",
|
|
"text": "x",
|
|
"conditions": ["x"],
|
|
"change": "changed",
|
|
"reason": "y",
|
|
},
|
|
}
|
|
)
|
|
|
|
assert "Baseline requirement FR-1 is missing" in requirements_check(dropped)[0]
|
|
assert (
|
|
"NFR-1 is marked added, but the Baseline has that id"
|
|
in (requirements_check(reused)[0])
|
|
)
|
|
assert (
|
|
"NFR-9 is marked changed, but the Baseline has no such requirement"
|
|
in (requirements_check(invented)[0])
|
|
)
|
|
|
|
|
|
def test_a_change_request_must_change_something():
|
|
assert (
|
|
"nothing is marked added, changed or removed"
|
|
in (requirements_check(revised())[0])
|
|
)
|
|
|
|
|
|
def test_a_removed_requirement_stays_on_record_and_asks_nothing_downstream():
|
|
from app.workflows.chain import read_list
|
|
|
|
call = revised(
|
|
**{"NFR-1": {"change": "removed", "reason": "No longer a requirement."}}
|
|
)
|
|
upstream = {"requirements": {"attachments": call.args["attachments"]}}
|
|
|
|
assert requirements_check(call) == ()
|
|
live = read_list(upstream, "requirements", "requirements.json")
|
|
assert [item["id"] for item in live] == ["FR-1"]
|
|
assert '"NFR-1"' in call.args["attachments"]["requirements.json"]
|
|
|
|
|
|
# --- step 4: later Specialists revise from the Baseline ---------------------------------
|
|
|
|
|
|
async def test_each_specialist_is_given_its_delivered_version_and_what_changed(
|
|
tmp_path,
|
|
):
|
|
await delivered(tmp_path)
|
|
runtime = FakeAgentRuntime(changed())
|
|
engine = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
)
|
|
|
|
await engine.start(
|
|
"Availability goes up.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
by_role = {run.role: run for run in runtime.runs}
|
|
analyst = by_role["requirements-analyst"].task.context["baseline"]
|
|
assert analyst["execution_id"] == "v1"
|
|
assert analyst["original_request"] == "Ingest webhooks"
|
|
assert "requirements.json" in analyst["deliverable"]["attachments"]
|
|
assert analyst["changes"] is None # nothing has changed yet
|
|
|
|
architect = by_role["target-architect"].task.context
|
|
assert architect["baseline"]["changes"]["requirements"] == [
|
|
{
|
|
"id": "NFR-1",
|
|
"change": "changed",
|
|
"reason": "The service is now customer-facing.",
|
|
"text": "99.99% availability.",
|
|
}
|
|
]
|
|
# Its working copy already holds the delivered graph.
|
|
assert architect["baseline"]["deliverable"]["attachments"][
|
|
"target-architecture.json"
|
|
]
|
|
assert "graph_read" in by_role["target-architect"].tools
|
|
# The guide is assembled afresh: the Baseline's is not handed back.
|
|
assert by_role["packager"].task.context["baseline"]["deliverable"] is None
|
|
|
|
|
|
async def test_a_description_whose_design_did_not_change_shape_is_carried_over(
|
|
tmp_path,
|
|
):
|
|
await delivered(tmp_path)
|
|
runtime = FakeAgentRuntime(changed())
|
|
engine = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
)
|
|
|
|
result = await engine.start(
|
|
"Availability goes up.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
# The same Elements and Relationships as the Baseline: the describer is not run.
|
|
assert "describer" not in {run.role for run in runtime.runs}
|
|
assert result.state["deliverables"]["architecture_description"]["content"]
|
|
|
|
|
|
async def test_a_code_authors_working_copy_starts_as_the_delivered_code(tmp_path):
|
|
await delivered(tmp_path)
|
|
seen: dict = {}
|
|
|
|
def author(task):
|
|
seen["working_copy"] = task.context["working_copy"]
|
|
return [sample_submission(K.DEPLOYMENT_GUIDANCE)]
|
|
|
|
engine = engine_for(tmp_path, changed(**{"iac-author": [author]}))
|
|
|
|
await engine.start(
|
|
"Availability goes up.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
# It is told nothing of a working copy on a first attempt; the tools hold it.
|
|
assert seen["working_copy"] is None
|
|
|
|
|
|
# --- what changed, worked out from the ids ------------------------------------------------
|
|
|
|
|
|
def versions(graph_change=None, requirement_change=None) -> tuple[dict, dict]:
|
|
"""A Baseline and a revised set of Deliverables, as data files."""
|
|
|
|
def deliverables() -> dict:
|
|
return {
|
|
kind: {"attachments": {name: SAMPLE_DATA[name]}}
|
|
for kind, name in (
|
|
("requirements", "requirements.json"),
|
|
("technical_decomposition", "components.json"),
|
|
("design_recommendations", "recommendations.json"),
|
|
("target_architecture", "target-architecture.json"),
|
|
)
|
|
}
|
|
|
|
before, after = deliverables(), deliverables()
|
|
change = requirement_change or RETENTION
|
|
after["requirements"]["attachments"] = revised(
|
|
**json.loads(json.dumps(change))
|
|
).args["attachments"]
|
|
if graph_change:
|
|
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
|
graph_change(graph)
|
|
after["target_architecture"]["attachments"] = {
|
|
"target-architecture.json": json.dumps(graph)
|
|
}
|
|
return before, after
|
|
|
|
|
|
def test_a_change_inside_the_reach_of_a_changed_requirement_is_requested():
|
|
from app.workflows.changes import change_summary
|
|
|
|
def tighten(graph):
|
|
graph["checks"][0]["statement"] = "The service runs at least three instances."
|
|
|
|
summary = change_summary(*versions(tighten))
|
|
|
|
assert [item["id"] for item in summary["requirements"]] == ["NFR-1"]
|
|
assert summary["checks"] == {"added": [], "changed": ["K-1"], "removed": []}
|
|
assert summary["elements"] == {"added": [], "changed": [], "removed": []}
|
|
assert summary["unrequested"] == {}
|
|
|
|
|
|
def test_a_change_no_changed_requirement_reaches_is_listed_as_unrequested():
|
|
from app.workflows.changes import change_summary
|
|
|
|
def wander(graph):
|
|
# K-2 comes from DR-1, which did not change, on E-2, which realises nothing
|
|
# the changed requirement touches. NFR-1 reaches K-1 and E-1 only.
|
|
graph["checks"][1]["statement"] = "The identity has no keys."
|
|
|
|
change = {
|
|
"NFR-1": {"change": "changed", "text": "99.99%.", "reason": "Customer-facing."}
|
|
}
|
|
before, after = versions(wander, change)
|
|
# NFR-1 is served by C-1, which E-1 realises; cut that link to isolate E-2.
|
|
components = [{"id": "C-1", "name": "Ingress", "requirements": ["FR-1"]}]
|
|
for version in (before, after):
|
|
version["technical_decomposition"]["attachments"] = {
|
|
"components.json": json.dumps(components)
|
|
}
|
|
|
|
summary = change_summary(before, after)
|
|
|
|
assert summary["checks"]["changed"] == ["K-2"]
|
|
assert summary["unrequested"] == {"checks": ["K-2"]}
|
|
|
|
|
|
def test_added_and_removed_items_are_found_by_id():
|
|
from app.workflows.changes import change_summary
|
|
|
|
def grow(graph):
|
|
graph["elements"].append(
|
|
{
|
|
"id": "E-3",
|
|
"name": "Second region service",
|
|
"component": "C-1",
|
|
"product": "Example Run",
|
|
"resource_type": "example_run_service",
|
|
}
|
|
)
|
|
graph["checks"] = [graph["checks"][0]] + [
|
|
{"id": "K-3", "applies_to": "E-3", "statement": "x", "source": "NFR-1"}
|
|
]
|
|
|
|
summary = change_summary(*versions(grow))
|
|
|
|
assert summary["elements"]["added"] == ["E-3"]
|
|
assert summary["checks"] == {"added": ["K-3"], "changed": [], "removed": ["K-2"]}
|
|
# K-2 came from DR-1 on E-2: nothing that changed reaches it.
|
|
assert summary["unrequested"] == {"checks": ["K-2"]}
|
|
|
|
|
|
# --- step 5: Approvals show the change --------------------------------------------------
|
|
|
|
|
|
async def test_an_approval_in_a_change_request_shows_what_that_phase_changes(
|
|
tmp_path,
|
|
):
|
|
from app.presentation import describe_prompt
|
|
|
|
await delivered(tmp_path)
|
|
engine = engine_for(tmp_path, changed())
|
|
|
|
first = await engine.start(
|
|
"Availability goes up.", customer_case=CASE, execution_id="v2", baseline="v1"
|
|
)
|
|
|
|
approval = first.prompt["approval"]
|
|
assert approval["phase"] == "requirements_discovery"
|
|
assert approval["baseline"] == "v1"
|
|
assert [item["id"] for item in approval["changes"]["requirements"]] == ["NFR-1"]
|
|
text = describe_prompt(first.prompt)
|
|
assert "This changes the architecture delivered in v1." in text
|
|
assert (
|
|
"- Requirement NFR-1 changed: 99.99% availability. (The service is now "
|
|
"customer-facing.)"
|
|
) in text
|
|
|
|
second = await engine.resume(
|
|
"v2", {"approval": {"phase": "requirements_discovery", "decision": "approve"}}
|
|
)
|
|
|
|
# The design did not change, and the Approval says so.
|
|
design = second.prompt["approval"]
|
|
assert design["phase"] == "solution_architecture" and design["changes"] == {}
|
|
assert "Nothing in this Phase differs from it." in describe_prompt(second.prompt)
|
|
|
|
|
|
async def test_an_ordinary_approval_says_nothing_of_changes(tmp_path):
|
|
first = await engine_for(tmp_path, once()).start("Ingest webhooks")
|
|
|
|
assert "changes" not in first.prompt["approval"]
|
|
|
|
|
|
# --- step 6: the change summary and the line of versions ----------------------------------
|
|
|
|
|
|
async def test_a_change_request_writes_changes_json_and_a_full_set_of_files(tmp_path):
|
|
from app.workflows.changes import change_summary
|
|
|
|
await delivered(tmp_path)
|
|
|
|
result = await engine_for(tmp_path, changed()).start(
|
|
"Availability goes up.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
case = tmp_path / "deliverables" / "cases" / CASE / "executions"
|
|
written = json.loads((case / "v2" / "changes.json").read_text())
|
|
assert [item["id"] for item in written["requirements"]] == ["NFR-1"]
|
|
assert written["unrequested"] == {}
|
|
# It matches a comparison of the two Executions' data files.
|
|
assert written == change_summary(
|
|
load_baseline(tmp_path, CASE, "v1")["deliverables"],
|
|
result.state["deliverables"],
|
|
)
|
|
# The Change request stands on its own: every Deliverable is written in full.
|
|
manifest = json.loads((case / "v2" / "execution.json").read_text())
|
|
assert manifest["changes"] == "changes.json"
|
|
assert set(manifest["deliverables"]) == {kind.value for kind in DEPENDENCY_GRAPH}
|
|
assert (case / "v2" / "solution-architecture-guide.md").is_file()
|
|
assert not (case / "v1" / "changes.json").exists()
|
|
|
|
|
|
# --- only the pillars a change touches are consulted again ------------------------------
|
|
|
|
|
|
async def test_a_change_request_starts_from_the_baselines_pillar_advice(tmp_path):
|
|
"""The design advisor is told which pillars the change affects, and has the
|
|
Baseline's advice for the rest."""
|
|
from app.adk.fake_runtime import FakeDecider
|
|
from app.adk.runtime import Answer
|
|
from app.tools.pillars import ADVICE_KEY
|
|
|
|
await delivered(tmp_path)
|
|
# A Baseline whose design advisor consulted two pillars.
|
|
folder = tmp_path / "deliverables" / "cases" / CASE / "executions" / "v1"
|
|
advice = {"security": "Use a dedicated identity.", "reliability": "Two zones."}
|
|
(folder / "working.json").write_text(
|
|
json.dumps({"design_recommendations": {ADVICE_KEY: advice}})
|
|
)
|
|
seen: dict = {}
|
|
|
|
def advisor(task):
|
|
seen["pillars_to_consult"] = task.context["pillars_to_consult"]
|
|
return [sample_submission(K.DESIGN_RECOMMENDATIONS)]
|
|
|
|
decider = FakeDecider(
|
|
{"security": Answer(0.3, 0.8), "reliability": Answer(1.9, 0.9)}
|
|
)
|
|
runtime = FakeAgentRuntime(changed(**{"design-advisor": [advisor]}))
|
|
engine = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
decider=decider,
|
|
)
|
|
|
|
result = await engine.start(
|
|
"Availability goes up.",
|
|
customer_case=CASE,
|
|
pre_approved=True,
|
|
execution_id="v2",
|
|
baseline="v1",
|
|
)
|
|
|
|
assert seen["pillars_to_consult"] == ["reliability"]
|
|
state, _, _ = decider.asked[0]
|
|
assert "Requirement NFR-1 changed: 99.99% availability." in state
|
|
# The advice is carried into the new Execution, and written for the next change.
|
|
assert result.state["working"]["design_recommendations"][ADVICE_KEY] == advice
|
|
assert json.loads((folder.parent / "v2" / "working.json").read_text()) == {
|
|
"design_recommendations": {ADVICE_KEY: advice}
|
|
}
|
|
|
|
|
|
async def test_a_first_consultation_is_given_no_hint(tmp_path):
|
|
seen: dict = {}
|
|
|
|
def advisor(task):
|
|
seen["pillars_to_consult"] = task.context["pillars_to_consult"]
|
|
return [sample_submission(K.DESIGN_RECOMMENDATIONS)]
|
|
|
|
from app.adk.fake_runtime import FakeDecider
|
|
|
|
decider = FakeDecider({})
|
|
turns = once()
|
|
turns["design-advisor"] = [advisor]
|
|
engine = ExecutionEngine(
|
|
runtime=FakeAgentRuntime(turns),
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
decider=decider,
|
|
)
|
|
|
|
await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert seen["pillars_to_consult"] is None and decider.asked == []
|
|
|
|
|
|
# --- changed, or only reworded? ---------------------------------------------------------
|
|
|
|
|
|
async def test_an_item_that_was_only_reworded_is_listed_apart_from_real_changes():
|
|
from app.adk.fake_runtime import FakeDecider
|
|
from app.adk.runtime import Answer
|
|
from app.tools.meaning import settle_meaning
|
|
from app.workflows.changes import change_summary
|
|
|
|
def revise(graph):
|
|
graph["checks"][0]["statement"] = "The service runs at least three instances."
|
|
graph["checks"][1]["statement"] = "This service alone uses the identity."
|
|
|
|
before, after = versions(revise)
|
|
summary = change_summary(before, after)
|
|
assert summary["checks"]["changed"] == ["K-1", "K-2"]
|
|
decider = FakeDecider({"K-1": Answer(0.95, 0.9), "K-2": Answer(0.1, 0.8)})
|
|
|
|
settled = await settle_meaning(decider, summary, before, after, scope="v2")
|
|
|
|
assert settled["checks"]["changed"] == ["K-1"]
|
|
assert settled["checks"]["reworded"] == ["K-2"]
|
|
assert settled["meaning"] == {"K-1": 0.95, "K-2": 0.1}
|
|
# K-2 was outside the reach of what changed; reworded, it is no longer unrequested.
|
|
assert settled["unrequested"] == {}
|
|
_, questions, scope = decider.asked[0]
|
|
assert scope == "v2"
|
|
assert "The identity is used by this service only." in questions["K-2"].instructions
|
|
assert "This service alone uses the identity." in questions["K-2"].instructions
|
|
# The summary given in is not altered.
|
|
assert summary["checks"]["changed"] == ["K-1", "K-2"]
|
|
|
|
|
|
async def test_without_a_decision_everything_that_differs_stays_changed():
|
|
from app.tools.meaning import settle_meaning
|
|
from app.workflows.changes import change_summary
|
|
|
|
def revise(graph):
|
|
graph["checks"][1]["statement"] = "This service alone uses the identity."
|
|
|
|
before, after = versions(revise)
|
|
summary = change_summary(before, after)
|
|
|
|
settled = await settle_meaning(NoDecider(), summary, before, after)
|
|
|
|
assert settled["checks"]["changed"] == ["K-2"]
|
|
assert "reworded" not in settled["checks"] and "meaning" not in settled
|