feat: prove the lessons, check overlays in evaluation, and settle the cost steps
Some checks failed
validation / verify (push) Failing after 13s

- Schema review: a dotted path is judged only when it starts at a name the
  resource type has, so a database flag or a service name is no longer read
  as a missing setting. Lesson L-04, which worked around that, is retired.
- Skill routing: in a Change request the question says what changed.
- Evaluation: a case may list code_standards the infrastructure code must
  meet; case 06 checks its overlay's regions, naming and labels.
- Model tiers (iteration 5 step 6): a lighter model for a role's first
  attempt, named in the provider bundle. Measured on case 03 and left off:
  Flash saved five cents on the IaC author and took twice the requests.
- Steps 7 and 9 decided against on the numbers, recorded in the plans.
- Plans: the runs of cases 05 and 06, and the first proof of the lessons on
  case 03 (no Revision, 22 minutes against 28, quality held).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 21:27:09 +01:00
parent 567aede5ca
commit e5e1647ff1
35 changed files with 497 additions and 39 deletions

View File

@@ -458,6 +458,14 @@ def make_execution_nodes(
role = dataclasses.replace(
role, skills=(*role.skills, *(n for n in named if n not in role.skills))
)
if role.first_attempt_model and task_input.get("first_attempt"):
logger.info(
"[%s] %s takes %s for a first attempt",
task_input["execution_id"],
role.name,
role.first_attempt_model,
)
role = dataclasses.replace(role, model=role.first_attempt_model)
kind = DeliverableKind(task_input["kind"])
working: dict[str, Any] = dict(task_input.get("working") or {})
upstream = task_input["context"].get("dependencies") or {}
@@ -861,6 +869,10 @@ def _specialist_input(
"role": spec.specialist,
"kind": kind.value,
"attempt": attempt,
# Nothing has been asked of this Deliverable again: no feedback on it, and
# no earlier attempt that ended without one. A lighter model may take it.
"first_attempt": not ledger.feedback(kind)
and (attempt == 1 or kind.value in deliverables),
"allow_cloud_read": state.get("allow_cloud_read"),
"working": dict(state.get("working", {}).get(kind.value, {})),
# Whether the user has been shown a draft of this Deliverable. Answering some
@@ -924,14 +936,9 @@ async def _pillar_hints(
) or baseline.get("working", {}).get(kind.value, {}).get(ADVICE_KEY)
if not remembered:
return {} # a first consultation: every pillar
change = "\n".join(ledger.feedback(kind))
if not change and baseline:
summary = change_summary(baseline.get("deliverables"), deliverables)
change = "\n".join(
f"Requirement {item['id']} {item['change']}: {item['text']} "
f"({item['reason']})"
for item in summary["requirements"]
)
change = "\n".join(ledger.feedback(kind)) or _requirement_changes(
baseline, deliverables
)
chosen = await pillars_touched(
decider,
list(remembered),
@@ -943,6 +950,19 @@ async def _pillar_hints(
return {} if chosen is None else {kind.value: chosen}
def _requirement_changes(
baseline: Mapping[str, Any], deliverables: Mapping[str, Mapping[str, Any]]
) -> str:
"""In a Change request, the requirements added, changed and removed, in words."""
if not baseline:
return ""
summary = change_summary(baseline.get("deliverables"), deliverables)
return "\n".join(
f"Requirement {item['id']} {item['change']}: {item['text']} ({item['reason']})"
for item in summary["requirements"]
)
async def _reading_hints(
state: ExecutionState,
ledger: Ledger,
@@ -970,9 +990,11 @@ async def _reading_hints(
for item in elements
)
changed = _requirement_changes(state.get("baseline") or {}, deliverables)
def task(kind: DeliverableKind) -> str:
feedback = list(ledger.feedback(kind))
doing = "Revise" if feedback else "Write"
doing = "Revise" if feedback or changed else "Write"
parts = [
(
f"{doing} the {kind.value.replace('_', ' ')} Deliverable of a cloud "
@@ -981,6 +1003,9 @@ async def _reading_hints(
]
if feedback:
parts.append("Feedback to address:\n" + "\n".join(feedback))
if changed:
# In a Change request the task is the change, not the whole design.
parts.append(f"What has changed since it was delivered:\n{changed}")
parts.append(
f"Elements of the Target architecture:\n{design}"
if design