feat: Change requests against a Baseline (ADR-0006)

A caller restates a request to say how requirements have changed, naming a
completed Execution of the same Customer case as the Baseline.

- app/baseline.py reads a Baseline from its files and refuses one that
  does not exist, did not complete or belongs to another case, before
  anything runs or is recorded. A Baseline is never modified.
- Requirements carry a change (unchanged, added, changed, removed) and a
  reason. Submission is refused when a Baseline requirement disappears, an
  id is reused, or an unchanged requirement's text differs. A removed
  requirement stays on record and asks nothing downstream.
- Each Specialist starts from its delivered version; the graph and the code
  are its working copy, and the description is carried over when the
  design's shape is the same.
- app/workflows/changes.py compares versions by id and traces what each
  changed requirement reaches; anything else that changed is listed as
  unrequested. Approvals show the change for their Phase, and changes.json
  is written with the full Deliverables.
- Intake no longer starts a message read as a change: it names the
  Execution to give as the Baseline.
- A fifth evaluation case runs a request and then a change to it, with
  four checks that the change is recorded, reaches Checks, stays contained
  and costs less than its Baseline.

Tested with scripted agents only; no Change request has run with real
models.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 11:38:15 +01:00
parent db28d02f56
commit 5ef434d00e
32 changed files with 1824 additions and 40 deletions

View File

@@ -77,6 +77,9 @@ class CaseSpec:
interactive: bool = False
# Upper bounds for "revisions", "seconds" and "tokens"; exceeding one is a regression.
thresholds: Mapping[str, float] = field(default_factory=dict)
# A change case: the request is run to completion, then this statement of what
# has changed is run as a Change request against it (ADR-0006).
change: str = ""
@dataclass(frozen=True)
@@ -118,6 +121,7 @@ def load_cases(path: Path) -> list[CaseSpec]:
),
interactive=bool(item.get("interactive", False)),
thresholds=dict(item.get("thresholds", {})),
change=str(item.get("change", "")),
)
for item in json.loads(path.read_text(encoding="utf-8"))
]
@@ -435,3 +439,107 @@ async def _quality(judge: Judge, recommendations: str) -> Check:
mean = sum(scores.values()) / len(QUALITY_RUBRIC)
detail = ", ".join(f"{key} {scores.get(key, 0):.2f}" for key in QUALITY_RUBRIC)
return Check(name, mean >= QUALITY_BAR, f"mean {mean:.2f}: {detail}")
# --- a Change request stays contained (iteration 7, step 7) ------------------------------
UNREQUESTED_SHARE = 0.1 # of the Baseline graph's items, changed without being reached
def score_change(
interaction: Interaction, baseline_tokens: float | None, tokens: float | None
) -> list[Check]:
"""Checks on a Change request against its Baseline.
They read what the Execution recorded: the requirements' own account of what
changed, and a comparison by id of the two versions' graphs.
"""
from app.workflows.changes import ADDED, CHANGED, REMOVED, change_summary
state = interaction.state
baseline = (state.get("baseline") or {}).get("deliverables") or {}
deliverables = state.get("deliverables", {})
summary = change_summary(baseline, deliverables)
requirements = summary["requirements"]
checks = [
Check(
"change_is_recorded",
bool(baseline) and bool(requirements),
", ".join(f"{item['id']} {item['change']}" for item in requirements)
or "no requirement is marked added, changed or removed",
)
]
graph = _graph_data(deliverables)
touched = set(summary["checks"][ADDED]) | set(summary["checks"][CHANGED])
removed_checks = set(summary["checks"][REMOVED])
old_sources = {
item.get("id"): item.get("source") for item in _graph_data(baseline)["checks"]
}
unreached = []
for item in requirements:
identifier = item["id"]
if item["change"] == REMOVED:
# What it asked for should go: a Check that came from it is removed.
reaches = any(old_sources.get(c) == identifier for c in removed_checks)
reaches = reaches or identifier not in old_sources.values()
else:
reaches = any(
check.get("id") in touched and check.get("source") == identifier
for check in graph["checks"]
)
if not reaches:
unreached.append(identifier)
checks.append(
Check(
"changed_requirements_reach_checks",
bool(requirements) and not unreached,
"no Check changed for: " + ", ".join(unreached) if unreached else "",
)
)
unrequested = [
f"{part} {identifier}"
for part, ids in summary["unrequested"].items()
for identifier in ids
]
size = sum(len(items) for items in _graph_data(baseline).values()) or 1
checks.append(
Check(
"change_is_contained",
len(unrequested) <= max(2, UNREQUESTED_SHARE * size),
f"{len(unrequested)} of {size} items changed without being reached"
+ (": " + ", ".join(unrequested[:12]) if unrequested else ""),
)
)
cheaper = (
baseline_tokens is not None and tokens is not None and tokens < baseline_tokens
)
checks.append(
Check(
"change_costs_less_than_baseline",
cheaper,
f"{tokens or 0:,.0f} tokens against {baseline_tokens or 0:,.0f}",
)
)
return checks
def _graph_data(
deliverables: Mapping[str, Mapping[str, Any]],
) -> dict[str, list[dict[str, Any]]]:
text = (
(deliverables.get(DeliverableKind.TARGET_ARCHITECTURE.value) or {})
.get("attachments", {})
.get("target-architecture.json", "")
)
try:
data = json.loads(text)
except ValueError:
data = {}
if not isinstance(data, dict):
data = {}
return {
part: [item for item in data.get(part, []) if isinstance(item, dict)]
for part in ("elements", "relationships", "checks")
}