feat: Change requests against a Baseline (ADR-0006)
A caller restates a request to say how requirements have changed, naming a completed Execution of the same Customer case as the Baseline. - app/baseline.py reads a Baseline from its files and refuses one that does not exist, did not complete or belongs to another case, before anything runs or is recorded. A Baseline is never modified. - Requirements carry a change (unchanged, added, changed, removed) and a reason. Submission is refused when a Baseline requirement disappears, an id is reused, or an unchanged requirement's text differs. A removed requirement stays on record and asks nothing downstream. - Each Specialist starts from its delivered version; the graph and the code are its working copy, and the description is carried over when the design's shape is the same. - app/workflows/changes.py compares versions by id and traces what each changed requirement reaches; anything else that changed is listed as unrequested. Approvals show the change for their Phase, and changes.json is written with the full Deliverables. - Intake no longer starts a message read as a change: it names the Execution to give as the Baseline. - A fifth evaluation case runs a request and then a change to it, with four checks that the change is recorded, reaches Checks, stays contained and costs less than its Baseline. Tested with scripted agents only; no Change request has run with real models. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
108
eval/scoring.py
108
eval/scoring.py
@@ -77,6 +77,9 @@ class CaseSpec:
|
||||
interactive: bool = False
|
||||
# Upper bounds for "revisions", "seconds" and "tokens"; exceeding one is a regression.
|
||||
thresholds: Mapping[str, float] = field(default_factory=dict)
|
||||
# A change case: the request is run to completion, then this statement of what
|
||||
# has changed is run as a Change request against it (ADR-0006).
|
||||
change: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -118,6 +121,7 @@ def load_cases(path: Path) -> list[CaseSpec]:
|
||||
),
|
||||
interactive=bool(item.get("interactive", False)),
|
||||
thresholds=dict(item.get("thresholds", {})),
|
||||
change=str(item.get("change", "")),
|
||||
)
|
||||
for item in json.loads(path.read_text(encoding="utf-8"))
|
||||
]
|
||||
@@ -435,3 +439,107 @@ async def _quality(judge: Judge, recommendations: str) -> Check:
|
||||
mean = sum(scores.values()) / len(QUALITY_RUBRIC)
|
||||
detail = ", ".join(f"{key} {scores.get(key, 0):.2f}" for key in QUALITY_RUBRIC)
|
||||
return Check(name, mean >= QUALITY_BAR, f"mean {mean:.2f}: {detail}")
|
||||
|
||||
|
||||
# --- a Change request stays contained (iteration 7, step 7) ------------------------------
|
||||
|
||||
UNREQUESTED_SHARE = 0.1 # of the Baseline graph's items, changed without being reached
|
||||
|
||||
|
||||
def score_change(
|
||||
interaction: Interaction, baseline_tokens: float | None, tokens: float | None
|
||||
) -> list[Check]:
|
||||
"""Checks on a Change request against its Baseline.
|
||||
|
||||
They read what the Execution recorded: the requirements' own account of what
|
||||
changed, and a comparison by id of the two versions' graphs.
|
||||
"""
|
||||
from app.workflows.changes import ADDED, CHANGED, REMOVED, change_summary
|
||||
|
||||
state = interaction.state
|
||||
baseline = (state.get("baseline") or {}).get("deliverables") or {}
|
||||
deliverables = state.get("deliverables", {})
|
||||
summary = change_summary(baseline, deliverables)
|
||||
requirements = summary["requirements"]
|
||||
checks = [
|
||||
Check(
|
||||
"change_is_recorded",
|
||||
bool(baseline) and bool(requirements),
|
||||
", ".join(f"{item['id']} {item['change']}" for item in requirements)
|
||||
or "no requirement is marked added, changed or removed",
|
||||
)
|
||||
]
|
||||
|
||||
graph = _graph_data(deliverables)
|
||||
touched = set(summary["checks"][ADDED]) | set(summary["checks"][CHANGED])
|
||||
removed_checks = set(summary["checks"][REMOVED])
|
||||
old_sources = {
|
||||
item.get("id"): item.get("source") for item in _graph_data(baseline)["checks"]
|
||||
}
|
||||
unreached = []
|
||||
for item in requirements:
|
||||
identifier = item["id"]
|
||||
if item["change"] == REMOVED:
|
||||
# What it asked for should go: a Check that came from it is removed.
|
||||
reaches = any(old_sources.get(c) == identifier for c in removed_checks)
|
||||
reaches = reaches or identifier not in old_sources.values()
|
||||
else:
|
||||
reaches = any(
|
||||
check.get("id") in touched and check.get("source") == identifier
|
||||
for check in graph["checks"]
|
||||
)
|
||||
if not reaches:
|
||||
unreached.append(identifier)
|
||||
checks.append(
|
||||
Check(
|
||||
"changed_requirements_reach_checks",
|
||||
bool(requirements) and not unreached,
|
||||
"no Check changed for: " + ", ".join(unreached) if unreached else "",
|
||||
)
|
||||
)
|
||||
|
||||
unrequested = [
|
||||
f"{part} {identifier}"
|
||||
for part, ids in summary["unrequested"].items()
|
||||
for identifier in ids
|
||||
]
|
||||
size = sum(len(items) for items in _graph_data(baseline).values()) or 1
|
||||
checks.append(
|
||||
Check(
|
||||
"change_is_contained",
|
||||
len(unrequested) <= max(2, UNREQUESTED_SHARE * size),
|
||||
f"{len(unrequested)} of {size} items changed without being reached"
|
||||
+ (": " + ", ".join(unrequested[:12]) if unrequested else ""),
|
||||
)
|
||||
)
|
||||
cheaper = (
|
||||
baseline_tokens is not None and tokens is not None and tokens < baseline_tokens
|
||||
)
|
||||
checks.append(
|
||||
Check(
|
||||
"change_costs_less_than_baseline",
|
||||
cheaper,
|
||||
f"{tokens or 0:,.0f} tokens against {baseline_tokens or 0:,.0f}",
|
||||
)
|
||||
)
|
||||
return checks
|
||||
|
||||
|
||||
def _graph_data(
|
||||
deliverables: Mapping[str, Mapping[str, Any]],
|
||||
) -> dict[str, list[dict[str, Any]]]:
|
||||
text = (
|
||||
(deliverables.get(DeliverableKind.TARGET_ARCHITECTURE.value) or {})
|
||||
.get("attachments", {})
|
||||
.get("target-architecture.json", "")
|
||||
)
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except ValueError:
|
||||
data = {}
|
||||
if not isinstance(data, dict):
|
||||
data = {}
|
||||
return {
|
||||
part: [item for item in data.get(part, []) if isinstance(item, dict)]
|
||||
for part in ("elements", "relationships", "checks")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user