feat: Change requests against a Baseline (ADR-0006)
A caller restates a request to say how requirements have changed, naming a completed Execution of the same Customer case as the Baseline. - app/baseline.py reads a Baseline from its files and refuses one that does not exist, did not complete or belongs to another case, before anything runs or is recorded. A Baseline is never modified. - Requirements carry a change (unchanged, added, changed, removed) and a reason. Submission is refused when a Baseline requirement disappears, an id is reused, or an unchanged requirement's text differs. A removed requirement stays on record and asks nothing downstream. - Each Specialist starts from its delivered version; the graph and the code are its working copy, and the description is carried over when the design's shape is the same. - app/workflows/changes.py compares versions by id and traces what each changed requirement reaches; anything else that changed is listed as unrequested. Approvals show the change for their Phase, and changes.json is written with the full Deliverables. - Intake no longer starts a message read as a change: it names the Execution to give as the Baseline. - A fifth evaluation case runs a request and then a change to it, with four checks that the change is recorded, reaches Checks, stays contained and costs less than its Baseline. Tested with scripted agents only; no Change request has run with real models. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -306,5 +306,66 @@
|
||||
"seconds": 3400,
|
||||
"tokens": 7500000
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "case-05-change-retention",
|
||||
"name": "A change to a delivered event-driven service: retention goes from 30 days to 7 years",
|
||||
"workflow_request": "Design an event-driven regional HTTP application on Google Cloud that accepts incoming webhooks, validates their signatures, durably enqueues payloads for asynchronous processing, and retains raw events for 30 days for replay and audit.",
|
||||
"change": "One thing has changed since this was delivered. Raw events must now be kept for 7 years instead of 30 days, to meet a new audit obligation, and must not be deletable during that time. Everything else stays as it is.",
|
||||
"required_products": [
|
||||
"Cloud Run",
|
||||
"Pub/Sub",
|
||||
"Cloud Storage"
|
||||
],
|
||||
"scripted_answers": [
|
||||
[
|
||||
"region|location|geograph",
|
||||
"A single region, europe-west2 (London). Data must stay in the UK."
|
||||
],
|
||||
[
|
||||
"availability|uptime|sla|rto|rpo|disaster|recover",
|
||||
"99.9% monthly availability. RTO 1 hour, RPO 5 minutes for accepted events. No multi-region DR in phase one."
|
||||
],
|
||||
[
|
||||
"traffic|volume|throughput|scale|peak|requests per|rps|payload",
|
||||
"Average 50 webhooks per second, peaks of 500 per second for 10 minutes. Payloads up to 256 KB."
|
||||
],
|
||||
[
|
||||
"latency|response time|performance",
|
||||
"Acknowledge each webhook within 300 ms at p95."
|
||||
],
|
||||
[
|
||||
"security|compliance|encrypt|privacy|gdpr|pci|access|identity|auth",
|
||||
"UK GDPR applies. Google-managed encryption keys are acceptable. Senders sign requests with HMAC-SHA256 shared secrets; only processing services may read events."
|
||||
],
|
||||
[
|
||||
"cost|budget|spend",
|
||||
"Under 1,500 GBP a month at average load; prefer pay-per-use."
|
||||
],
|
||||
[
|
||||
"current|existing|today|on-prem|migrat|legacy",
|
||||
"This is a new workload; nothing runs today."
|
||||
],
|
||||
[
|
||||
"depend|integrat|downstream|upstream|consum",
|
||||
"Processed events go to an existing internal REST API that is reachable over HTTPS; it can be slow or unavailable for minutes."
|
||||
],
|
||||
[
|
||||
"operat|monitor|logging|alert|team|support",
|
||||
"A small platform team of three uses Terraform and wants managed services and Cloud Monitoring alerts."
|
||||
],
|
||||
[
|
||||
"sustainab|carbon",
|
||||
"Prefer low-carbon regions when there is no other constraint, but the UK residency requirement wins."
|
||||
],
|
||||
[
|
||||
"retention|replay|audit|archive",
|
||||
"Keep raw events 30 days, then delete. Replay must be possible for any event in that window."
|
||||
],
|
||||
[
|
||||
"approve|confirm|proceed|look right|any changes|happy",
|
||||
"Approved. Please proceed."
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
108
eval/scoring.py
108
eval/scoring.py
@@ -77,6 +77,9 @@ class CaseSpec:
|
||||
interactive: bool = False
|
||||
# Upper bounds for "revisions", "seconds" and "tokens"; exceeding one is a regression.
|
||||
thresholds: Mapping[str, float] = field(default_factory=dict)
|
||||
# A change case: the request is run to completion, then this statement of what
|
||||
# has changed is run as a Change request against it (ADR-0006).
|
||||
change: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -118,6 +121,7 @@ def load_cases(path: Path) -> list[CaseSpec]:
|
||||
),
|
||||
interactive=bool(item.get("interactive", False)),
|
||||
thresholds=dict(item.get("thresholds", {})),
|
||||
change=str(item.get("change", "")),
|
||||
)
|
||||
for item in json.loads(path.read_text(encoding="utf-8"))
|
||||
]
|
||||
@@ -435,3 +439,107 @@ async def _quality(judge: Judge, recommendations: str) -> Check:
|
||||
mean = sum(scores.values()) / len(QUALITY_RUBRIC)
|
||||
detail = ", ".join(f"{key} {scores.get(key, 0):.2f}" for key in QUALITY_RUBRIC)
|
||||
return Check(name, mean >= QUALITY_BAR, f"mean {mean:.2f}: {detail}")
|
||||
|
||||
|
||||
# --- a Change request stays contained (iteration 7, step 7) ------------------------------
|
||||
|
||||
UNREQUESTED_SHARE = 0.1 # of the Baseline graph's items, changed without being reached
|
||||
|
||||
|
||||
def score_change(
|
||||
interaction: Interaction, baseline_tokens: float | None, tokens: float | None
|
||||
) -> list[Check]:
|
||||
"""Checks on a Change request against its Baseline.
|
||||
|
||||
They read what the Execution recorded: the requirements' own account of what
|
||||
changed, and a comparison by id of the two versions' graphs.
|
||||
"""
|
||||
from app.workflows.changes import ADDED, CHANGED, REMOVED, change_summary
|
||||
|
||||
state = interaction.state
|
||||
baseline = (state.get("baseline") or {}).get("deliverables") or {}
|
||||
deliverables = state.get("deliverables", {})
|
||||
summary = change_summary(baseline, deliverables)
|
||||
requirements = summary["requirements"]
|
||||
checks = [
|
||||
Check(
|
||||
"change_is_recorded",
|
||||
bool(baseline) and bool(requirements),
|
||||
", ".join(f"{item['id']} {item['change']}" for item in requirements)
|
||||
or "no requirement is marked added, changed or removed",
|
||||
)
|
||||
]
|
||||
|
||||
graph = _graph_data(deliverables)
|
||||
touched = set(summary["checks"][ADDED]) | set(summary["checks"][CHANGED])
|
||||
removed_checks = set(summary["checks"][REMOVED])
|
||||
old_sources = {
|
||||
item.get("id"): item.get("source") for item in _graph_data(baseline)["checks"]
|
||||
}
|
||||
unreached = []
|
||||
for item in requirements:
|
||||
identifier = item["id"]
|
||||
if item["change"] == REMOVED:
|
||||
# What it asked for should go: a Check that came from it is removed.
|
||||
reaches = any(old_sources.get(c) == identifier for c in removed_checks)
|
||||
reaches = reaches or identifier not in old_sources.values()
|
||||
else:
|
||||
reaches = any(
|
||||
check.get("id") in touched and check.get("source") == identifier
|
||||
for check in graph["checks"]
|
||||
)
|
||||
if not reaches:
|
||||
unreached.append(identifier)
|
||||
checks.append(
|
||||
Check(
|
||||
"changed_requirements_reach_checks",
|
||||
bool(requirements) and not unreached,
|
||||
"no Check changed for: " + ", ".join(unreached) if unreached else "",
|
||||
)
|
||||
)
|
||||
|
||||
unrequested = [
|
||||
f"{part} {identifier}"
|
||||
for part, ids in summary["unrequested"].items()
|
||||
for identifier in ids
|
||||
]
|
||||
size = sum(len(items) for items in _graph_data(baseline).values()) or 1
|
||||
checks.append(
|
||||
Check(
|
||||
"change_is_contained",
|
||||
len(unrequested) <= max(2, UNREQUESTED_SHARE * size),
|
||||
f"{len(unrequested)} of {size} items changed without being reached"
|
||||
+ (": " + ", ".join(unrequested[:12]) if unrequested else ""),
|
||||
)
|
||||
)
|
||||
cheaper = (
|
||||
baseline_tokens is not None and tokens is not None and tokens < baseline_tokens
|
||||
)
|
||||
checks.append(
|
||||
Check(
|
||||
"change_costs_less_than_baseline",
|
||||
cheaper,
|
||||
f"{tokens or 0:,.0f} tokens against {baseline_tokens or 0:,.0f}",
|
||||
)
|
||||
)
|
||||
return checks
|
||||
|
||||
|
||||
def _graph_data(
|
||||
deliverables: Mapping[str, Mapping[str, Any]],
|
||||
) -> dict[str, list[dict[str, Any]]]:
|
||||
text = (
|
||||
(deliverables.get(DeliverableKind.TARGET_ARCHITECTURE.value) or {})
|
||||
.get("attachments", {})
|
||||
.get("target-architecture.json", "")
|
||||
)
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except ValueError:
|
||||
data = {}
|
||||
if not isinstance(data, dict):
|
||||
data = {}
|
||||
return {
|
||||
part: [item for item in data.get(part, []) if isinstance(item, dict)]
|
||||
for part in ("elements", "relationships", "checks")
|
||||
}
|
||||
|
||||
@@ -41,9 +41,11 @@ from eval.judge import DeciderJudge, GeminiJudge, Judge
|
||||
from eval.scoring import (
|
||||
CaseResult,
|
||||
CaseSpec,
|
||||
Check,
|
||||
checklist_items,
|
||||
load_cases,
|
||||
regressions,
|
||||
score_change,
|
||||
score_execution,
|
||||
template_headings,
|
||||
)
|
||||
@@ -179,7 +181,33 @@ async def run_case(
|
||||
settings, runtime=runtime, recorder=recorder, output_root=run_dir
|
||||
) as assembly:
|
||||
execution_id = f"{case.id}-{datetime.now(UTC):%H%M%S}"
|
||||
if case.interactive:
|
||||
baseline_tokens: float | None = None
|
||||
baseline_seconds: float | None = None
|
||||
if case.change:
|
||||
# A change case: deliver the request, then run the change against it.
|
||||
base = await assembly.engine.start(
|
||||
case.workflow_request,
|
||||
customer_case=case.id,
|
||||
pre_approved=True,
|
||||
allow_cloud_read=False,
|
||||
execution_id=f"{execution_id}-base",
|
||||
scripted_answers=case.scripted_answers,
|
||||
)
|
||||
baseline_tokens = token_report(adk_runtime.usage)["total"].get("total")
|
||||
baseline_seconds = round(time.monotonic() - started, 1)
|
||||
if base.status != "completed":
|
||||
interaction = base # nothing to change: score what there is
|
||||
else:
|
||||
interaction = await assembly.engine.start(
|
||||
case.change,
|
||||
customer_case=case.id,
|
||||
pre_approved=True,
|
||||
allow_cloud_read=False,
|
||||
execution_id=execution_id,
|
||||
scripted_answers=case.scripted_answers,
|
||||
baseline=base.execution_id,
|
||||
)
|
||||
elif case.interactive:
|
||||
interaction = await run_with_scripted_user(
|
||||
assembly.engine, case, execution_id
|
||||
)
|
||||
@@ -220,6 +248,15 @@ async def run_case(
|
||||
)
|
||||
ledger = interaction.state.get("ledger", {})
|
||||
tokens = token_report(adk_runtime.usage)
|
||||
change_checks: tuple[Check, ...] = ()
|
||||
if case.change:
|
||||
total = tokens["total"].get("total")
|
||||
change_tokens = (
|
||||
total - baseline_tokens
|
||||
if total is not None and baseline_tokens is not None
|
||||
else None
|
||||
)
|
||||
change_checks = tuple(score_change(interaction, baseline_tokens, change_tokens))
|
||||
revisions = {
|
||||
kind: item["revisions"]
|
||||
for kind, item in ledger.get("deliverables", {}).items()
|
||||
@@ -250,7 +287,17 @@ async def run_case(
|
||||
"conversation": interaction.state.get("conversation", []),
|
||||
"output_dir": str(execution_dir(run_dir, interaction.execution_id, case.id)),
|
||||
}
|
||||
return CaseResult(result.case_id, result.status, result.checks, details)
|
||||
if case.change:
|
||||
details["baseline"] = {
|
||||
"execution_id": (interaction.state.get("baseline") or {}).get(
|
||||
"execution_id"
|
||||
),
|
||||
"seconds": baseline_seconds,
|
||||
"tokens": baseline_tokens,
|
||||
}
|
||||
return CaseResult(
|
||||
result.case_id, result.status, (*result.checks, *change_checks), details
|
||||
)
|
||||
|
||||
|
||||
def make_judge(settings: Settings) -> Judge:
|
||||
|
||||
Reference in New Issue
Block a user