feat: Change requests against a Baseline (ADR-0006)
A caller restates a request to say how requirements have changed, naming a completed Execution of the same Customer case as the Baseline. - app/baseline.py reads a Baseline from its files and refuses one that does not exist, did not complete or belongs to another case, before anything runs or is recorded. A Baseline is never modified. - Requirements carry a change (unchanged, added, changed, removed) and a reason. Submission is refused when a Baseline requirement disappears, an id is reused, or an unchanged requirement's text differs. A removed requirement stays on record and asks nothing downstream. - Each Specialist starts from its delivered version; the graph and the code are its working copy, and the description is carried over when the design's shape is the same. - app/workflows/changes.py compares versions by id and traces what each changed requirement reaches; anything else that changed is listed as unrequested. Approvals show the change for their Phase, and changes.json is written with the full Deliverables. - Intake no longer starts a message read as a change: it names the Execution to give as the Baseline. - A fifth evaluation case runs a request and then a change to it, with four checks that the change is recorded, reaches Checks, stays contained and costs less than its Baseline. Tested with scripted agents only; no Change request has run with real models. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
"""Scoring a finished Execution against a benchmark case, without models or network."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from app.adk.fake_runtime import SAMPLE_DATA
|
||||
@@ -614,3 +615,133 @@ def test_calibration_without_labels_still_measures_stability():
|
||||
assert result["labelled"] == 0
|
||||
assert result["judges"]["jev"]["agreement_with_labels"] is None
|
||||
assert result["judges"]["jev"]["stability"] == 1.0
|
||||
|
||||
|
||||
# --- a Change request stays contained ---------------------------------------------------
|
||||
|
||||
|
||||
def change_interaction(graph_change=None, requirement=None) -> Interaction:
|
||||
from app.adk.fake_runtime import SAMPLE_DATA
|
||||
|
||||
names = {
|
||||
"requirements": "requirements.json",
|
||||
"technical_decomposition": "components.json",
|
||||
"design_recommendations": "recommendations.json",
|
||||
"target_architecture": "target-architecture.json",
|
||||
}
|
||||
|
||||
def version() -> dict:
|
||||
return {
|
||||
kind: {"attachments": {name: SAMPLE_DATA[name]}}
|
||||
for kind, name in names.items()
|
||||
}
|
||||
|
||||
baseline, current = version(), version()
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
for item in requirements:
|
||||
item["change"] = "unchanged"
|
||||
if requirement is not False:
|
||||
requirements[1].update(
|
||||
requirement
|
||||
or {"change": "changed", "text": "99.99%.", "reason": "Customer-facing."}
|
||||
)
|
||||
current["requirements"]["attachments"] = {
|
||||
"requirements.json": json.dumps(requirements)
|
||||
}
|
||||
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
if graph_change:
|
||||
graph_change(graph)
|
||||
current["target_architecture"]["attachments"] = {
|
||||
"target-architecture.json": json.dumps(graph)
|
||||
}
|
||||
state = {"baseline": {"deliverables": baseline}, "deliverables": current}
|
||||
return Interaction("v2", "completed", None, state)
|
||||
|
||||
|
||||
def results(checks) -> dict[str, bool]:
|
||||
return {check.name: check.passed for check in checks}
|
||||
|
||||
|
||||
def test_a_contained_change_passes_every_change_check():
|
||||
from eval.scoring import score_change
|
||||
|
||||
def tighten(graph):
|
||||
graph["checks"][0]["statement"] = "The service runs at least three instances."
|
||||
|
||||
checks = score_change(change_interaction(tighten), 4_000_000, 1_500_000)
|
||||
|
||||
assert results(checks) == {
|
||||
"change_is_recorded": True,
|
||||
"changed_requirements_reach_checks": True,
|
||||
"change_is_contained": True,
|
||||
"change_costs_less_than_baseline": True,
|
||||
}
|
||||
|
||||
|
||||
def test_a_changed_requirement_that_changes_no_check_fails():
|
||||
from eval.scoring import score_change
|
||||
|
||||
checks = score_change(change_interaction(), 4_000_000, 1_500_000)
|
||||
|
||||
by_name = {check.name: check for check in checks}
|
||||
assert not by_name["changed_requirements_reach_checks"].passed
|
||||
assert by_name["changed_requirements_reach_checks"].detail == (
|
||||
"no Check changed for: NFR-1"
|
||||
)
|
||||
|
||||
|
||||
def test_a_change_that_cost_more_than_the_baseline_or_changed_nothing_fails():
|
||||
from eval.scoring import score_change
|
||||
|
||||
dearer = results(score_change(change_interaction(), 2_000_000, 3_000_000))
|
||||
nothing = results(
|
||||
score_change(change_interaction(requirement=False), 2_000_000, 1_000_000)
|
||||
)
|
||||
|
||||
assert not dearer["change_costs_less_than_baseline"]
|
||||
assert not nothing["change_is_recorded"]
|
||||
|
||||
|
||||
def test_many_changes_nobody_asked_for_fail_containment():
|
||||
from eval.scoring import score_change
|
||||
|
||||
def rewrite(graph):
|
||||
# Everything about the identity changes; NFR-1 reaches none of it.
|
||||
graph["elements"][1]["name"] = "A different identity"
|
||||
graph["checks"][1]["statement"] = "Something else."
|
||||
graph["elements"].append(
|
||||
{
|
||||
"id": "E-9",
|
||||
"name": "An extra key",
|
||||
"product": "Example KMS",
|
||||
"resource_type": "example_key",
|
||||
}
|
||||
)
|
||||
graph["checks"].append(
|
||||
{"id": "K-9", "applies_to": "E-9", "statement": "x", "source": "DR-1"}
|
||||
)
|
||||
|
||||
components = [{"id": "C-1", "name": "Ingress", "requirements": ["FR-1"]}]
|
||||
interaction = change_interaction(rewrite)
|
||||
for version in (
|
||||
interaction.state["baseline"]["deliverables"],
|
||||
interaction.state["deliverables"],
|
||||
):
|
||||
version["technical_decomposition"]["attachments"] = {
|
||||
"components.json": json.dumps(components)
|
||||
}
|
||||
|
||||
by_name = {check.name: check for check in score_change(interaction, 4, 1)}
|
||||
|
||||
assert not by_name["change_is_contained"].passed
|
||||
assert by_name["change_is_contained"].detail.startswith("4 of 5 items changed")
|
||||
|
||||
|
||||
def test_the_change_case_loads_with_its_statement_of_change():
|
||||
from eval.scoring import load_cases
|
||||
|
||||
cases = load_cases(Path(__file__).parents[2] / "eval/datasets/solution_cases.json")
|
||||
case = next(c for c in cases if c.id == "case-05-change-retention")
|
||||
|
||||
assert "7 years" in case.change
|
||||
assert all(not c.change for c in cases if c.id != case.id)
|
||||
|
||||
Reference in New Issue
Block a user