feat: Change requests against a Baseline (ADR-0006)
A caller restates a request to say how requirements have changed, naming a completed Execution of the same Customer case as the Baseline. - app/baseline.py reads a Baseline from its files and refuses one that does not exist, did not complete or belongs to another case, before anything runs or is recorded. A Baseline is never modified. - Requirements carry a change (unchanged, added, changed, removed) and a reason. Submission is refused when a Baseline requirement disappears, an id is reused, or an unchanged requirement's text differs. A removed requirement stays on record and asks nothing downstream. - Each Specialist starts from its delivered version; the graph and the code are its working copy, and the description is carried over when the design's shape is the same. - app/workflows/changes.py compares versions by id and traces what each changed requirement reaches; anything else that changed is listed as unrequested. Approvals show the change for their Phase, and changes.json is written with the full Deliverables. - Intake no longer starts a message read as a change: it names the Execution to give as the Baseline. - A fifth evaluation case runs a request and then a change to it, with four checks that the change is recorded, reaches Checks, stays contained and costs less than its Baseline. Tested with scripted agents only; no Change request has run with real models. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -41,9 +41,11 @@ from eval.judge import DeciderJudge, GeminiJudge, Judge
|
||||
from eval.scoring import (
|
||||
CaseResult,
|
||||
CaseSpec,
|
||||
Check,
|
||||
checklist_items,
|
||||
load_cases,
|
||||
regressions,
|
||||
score_change,
|
||||
score_execution,
|
||||
template_headings,
|
||||
)
|
||||
@@ -179,7 +181,33 @@ async def run_case(
|
||||
settings, runtime=runtime, recorder=recorder, output_root=run_dir
|
||||
) as assembly:
|
||||
execution_id = f"{case.id}-{datetime.now(UTC):%H%M%S}"
|
||||
if case.interactive:
|
||||
baseline_tokens: float | None = None
|
||||
baseline_seconds: float | None = None
|
||||
if case.change:
|
||||
# A change case: deliver the request, then run the change against it.
|
||||
base = await assembly.engine.start(
|
||||
case.workflow_request,
|
||||
customer_case=case.id,
|
||||
pre_approved=True,
|
||||
allow_cloud_read=False,
|
||||
execution_id=f"{execution_id}-base",
|
||||
scripted_answers=case.scripted_answers,
|
||||
)
|
||||
baseline_tokens = token_report(adk_runtime.usage)["total"].get("total")
|
||||
baseline_seconds = round(time.monotonic() - started, 1)
|
||||
if base.status != "completed":
|
||||
interaction = base # nothing to change: score what there is
|
||||
else:
|
||||
interaction = await assembly.engine.start(
|
||||
case.change,
|
||||
customer_case=case.id,
|
||||
pre_approved=True,
|
||||
allow_cloud_read=False,
|
||||
execution_id=execution_id,
|
||||
scripted_answers=case.scripted_answers,
|
||||
baseline=base.execution_id,
|
||||
)
|
||||
elif case.interactive:
|
||||
interaction = await run_with_scripted_user(
|
||||
assembly.engine, case, execution_id
|
||||
)
|
||||
@@ -220,6 +248,15 @@ async def run_case(
|
||||
)
|
||||
ledger = interaction.state.get("ledger", {})
|
||||
tokens = token_report(adk_runtime.usage)
|
||||
change_checks: tuple[Check, ...] = ()
|
||||
if case.change:
|
||||
total = tokens["total"].get("total")
|
||||
change_tokens = (
|
||||
total - baseline_tokens
|
||||
if total is not None and baseline_tokens is not None
|
||||
else None
|
||||
)
|
||||
change_checks = tuple(score_change(interaction, baseline_tokens, change_tokens))
|
||||
revisions = {
|
||||
kind: item["revisions"]
|
||||
for kind, item in ledger.get("deliverables", {}).items()
|
||||
@@ -250,7 +287,17 @@ async def run_case(
|
||||
"conversation": interaction.state.get("conversation", []),
|
||||
"output_dir": str(execution_dir(run_dir, interaction.execution_id, case.id)),
|
||||
}
|
||||
return CaseResult(result.case_id, result.status, result.checks, details)
|
||||
if case.change:
|
||||
details["baseline"] = {
|
||||
"execution_id": (interaction.state.get("baseline") or {}).get(
|
||||
"execution_id"
|
||||
),
|
||||
"seconds": baseline_seconds,
|
||||
"tokens": baseline_tokens,
|
||||
}
|
||||
return CaseResult(
|
||||
result.case_id, result.status, (*result.checks, *change_checks), details
|
||||
)
|
||||
|
||||
|
||||
def make_judge(settings: Settings) -> Judge:
|
||||
|
||||
Reference in New Issue
Block a user