feat: prove the lessons, check overlays in evaluation, and settle the cost steps
Some checks failed
validation / verify (push) Failing after 13s
Some checks failed
validation / verify (push) Failing after 13s
- Schema review: a dotted path is judged only when it starts at a name the resource type has, so a database flag or a service name is no longer read as a missing setting. Lesson L-04, which worked around that, is retired. - Skill routing: in a Change request the question says what changed. - Evaluation: a case may list code_standards the infrastructure code must meet; case 06 checks its overlay's regions, naming and labels. - Model tiers (iteration 5 step 6): a lighter model for a role's first attempt, named in the provider bundle. Measured on case 03 and left off: Flash saved five cents on the IaC author and took twice the requests. - Steps 7 and 9 decided against on the numbers, recorded in the plans. - Plans: the runs of cases 05 and 06, and the first proof of the lessons on case 03 (no Revision, 22 minutes against 28, quality held). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -43,9 +43,11 @@ def load_roles(
|
||||
specialist_budget: int = 15,
|
||||
include_preview_mcp: bool = True,
|
||||
lessons: bool = True,
|
||||
model_tiers: bool = False,
|
||||
) -> dict[str, RoleSpec]:
|
||||
"""`lessons` off leaves the lessons Skill out of every role, so an evaluation
|
||||
can run with and without what has been learned."""
|
||||
can run with and without what has been learned. `model_tiers` on gives a role
|
||||
the lighter model the bundle names for its first attempt."""
|
||||
specialists = {spec.specialist: kind for kind, spec in DEPENDENCY_GRAPH.items()}
|
||||
required = {ORCHESTRATOR, PILLAR_ADVISOR, REPLY_WRITER, *specialists}
|
||||
optional = {LESSON_CURATOR}
|
||||
@@ -74,6 +76,13 @@ def load_roles(
|
||||
roles[name] = dataclasses.replace(
|
||||
roles[name], skills=(*roles[name].skills, bundle.lessons_skill)
|
||||
)
|
||||
if model_tiers:
|
||||
for name, model in bundle.models.first_attempt.items():
|
||||
if name not in roles:
|
||||
raise RoleConfigError(
|
||||
f"'models.first_attempt' names {name!r}, which is not a role"
|
||||
)
|
||||
roles[name] = dataclasses.replace(roles[name], first_attempt_model=model)
|
||||
unexpected = sorted(set(roles) - required - optional)
|
||||
if unexpected:
|
||||
raise RoleConfigError(
|
||||
|
||||
@@ -34,6 +34,9 @@ class RoleSpec:
|
||||
model: str | None = None
|
||||
# Seconds one model request may take; None takes the runtime's default.
|
||||
model_timeout_seconds: float | None = None
|
||||
# A lighter model for the first attempt at a Deliverable, when the bundle
|
||||
# names one and model tiers are on (iteration 5 step 6).
|
||||
first_attempt_model: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -379,6 +379,7 @@ async def open_assembly(
|
||||
specialist_budget=settings.SPECIALIST_BUDGET,
|
||||
include_preview_mcp=settings.MCP_ENABLE_PREVIEW,
|
||||
lessons=settings.LESSONS_ENABLED,
|
||||
model_tiers=settings.MODEL_TIERS,
|
||||
)
|
||||
overlays = discover_overlays(settings, bundle)
|
||||
if runtime is None:
|
||||
|
||||
@@ -64,6 +64,10 @@ class Settings(BaseSettings):
|
||||
# Give Specialists the lessons Skill. Off runs without anything learned, which
|
||||
# is how the evaluation shows whether a lesson helps.
|
||||
LESSONS_ENABLED: bool = True
|
||||
# Give a role the lighter model its provider bundle names for a first attempt
|
||||
# (`models.first_attempt`). An attempt that follows feedback, or an attempt
|
||||
# that produced nothing, takes the role's own model. Off until measured.
|
||||
MODEL_TIERS: bool = False
|
||||
# With a user present, a Specialist's first submission of a Deliverable is refused
|
||||
# until it has put its draft to them. Off: Specialists are only told to.
|
||||
CONFIRM_DRAFTS: bool = True
|
||||
|
||||
@@ -458,6 +458,14 @@ def make_execution_nodes(
|
||||
role = dataclasses.replace(
|
||||
role, skills=(*role.skills, *(n for n in named if n not in role.skills))
|
||||
)
|
||||
if role.first_attempt_model and task_input.get("first_attempt"):
|
||||
logger.info(
|
||||
"[%s] %s takes %s for a first attempt",
|
||||
task_input["execution_id"],
|
||||
role.name,
|
||||
role.first_attempt_model,
|
||||
)
|
||||
role = dataclasses.replace(role, model=role.first_attempt_model)
|
||||
kind = DeliverableKind(task_input["kind"])
|
||||
working: dict[str, Any] = dict(task_input.get("working") or {})
|
||||
upstream = task_input["context"].get("dependencies") or {}
|
||||
@@ -861,6 +869,10 @@ def _specialist_input(
|
||||
"role": spec.specialist,
|
||||
"kind": kind.value,
|
||||
"attempt": attempt,
|
||||
# Nothing has been asked of this Deliverable again: no feedback on it, and
|
||||
# no earlier attempt that ended without one. A lighter model may take it.
|
||||
"first_attempt": not ledger.feedback(kind)
|
||||
and (attempt == 1 or kind.value in deliverables),
|
||||
"allow_cloud_read": state.get("allow_cloud_read"),
|
||||
"working": dict(state.get("working", {}).get(kind.value, {})),
|
||||
# Whether the user has been shown a draft of this Deliverable. Answering some
|
||||
@@ -924,14 +936,9 @@ async def _pillar_hints(
|
||||
) or baseline.get("working", {}).get(kind.value, {}).get(ADVICE_KEY)
|
||||
if not remembered:
|
||||
return {} # a first consultation: every pillar
|
||||
change = "\n".join(ledger.feedback(kind))
|
||||
if not change and baseline:
|
||||
summary = change_summary(baseline.get("deliverables"), deliverables)
|
||||
change = "\n".join(
|
||||
f"Requirement {item['id']} {item['change']}: {item['text']} "
|
||||
f"({item['reason']})"
|
||||
for item in summary["requirements"]
|
||||
)
|
||||
change = "\n".join(ledger.feedback(kind)) or _requirement_changes(
|
||||
baseline, deliverables
|
||||
)
|
||||
chosen = await pillars_touched(
|
||||
decider,
|
||||
list(remembered),
|
||||
@@ -943,6 +950,19 @@ async def _pillar_hints(
|
||||
return {} if chosen is None else {kind.value: chosen}
|
||||
|
||||
|
||||
def _requirement_changes(
|
||||
baseline: Mapping[str, Any], deliverables: Mapping[str, Mapping[str, Any]]
|
||||
) -> str:
|
||||
"""In a Change request, the requirements added, changed and removed, in words."""
|
||||
if not baseline:
|
||||
return ""
|
||||
summary = change_summary(baseline.get("deliverables"), deliverables)
|
||||
return "\n".join(
|
||||
f"Requirement {item['id']} {item['change']}: {item['text']} ({item['reason']})"
|
||||
for item in summary["requirements"]
|
||||
)
|
||||
|
||||
|
||||
async def _reading_hints(
|
||||
state: ExecutionState,
|
||||
ledger: Ledger,
|
||||
@@ -970,9 +990,11 @@ async def _reading_hints(
|
||||
for item in elements
|
||||
)
|
||||
|
||||
changed = _requirement_changes(state.get("baseline") or {}, deliverables)
|
||||
|
||||
def task(kind: DeliverableKind) -> str:
|
||||
feedback = list(ledger.feedback(kind))
|
||||
doing = "Revise" if feedback else "Write"
|
||||
doing = "Revise" if feedback or changed else "Write"
|
||||
parts = [
|
||||
(
|
||||
f"{doing} the {kind.value.replace('_', ' ')} Deliverable of a cloud "
|
||||
@@ -981,6 +1003,9 @@ async def _reading_hints(
|
||||
]
|
||||
if feedback:
|
||||
parts.append("Feedback to address:\n" + "\n".join(feedback))
|
||||
if changed:
|
||||
# In a Change request the task is the change, not the whole design.
|
||||
parts.append(f"What has changed since it was delivered:\n{changed}")
|
||||
parts.append(
|
||||
f"Elements of the Target architecture:\n{design}"
|
||||
if design
|
||||
|
||||
@@ -45,6 +45,9 @@ class ModelConfig:
|
||||
# Seconds one model request may take, for the roles whose replies are long. A
|
||||
# role not named here gets the runtime's default.
|
||||
timeouts: Mapping[str, float] = field(default_factory=dict)
|
||||
# A lighter model for a role's first attempt at a Deliverable. Only for a role
|
||||
# whose work something else checks; an attempt after feedback takes its own.
|
||||
first_attempt: Mapping[str, str] = field(default_factory=dict)
|
||||
|
||||
def for_role(self, role: str) -> str:
|
||||
return self.roles.get(role, self.default)
|
||||
@@ -131,6 +134,10 @@ def load_provider(name: str, root: Path | None = None) -> ProviderBundle:
|
||||
default=models["default"],
|
||||
roles=dict(models.get("roles") or {}),
|
||||
timeouts=timeouts,
|
||||
first_attempt={
|
||||
str(role): str(model)
|
||||
for role, model in (models.get("first_attempt") or {}).items()
|
||||
},
|
||||
)
|
||||
|
||||
terraform_providers = {}
|
||||
|
||||
@@ -64,6 +64,20 @@ def _names(block: Mapping[str, Any]) -> set[str]:
|
||||
return names
|
||||
|
||||
|
||||
def _is_missing(setting: str, known: set[str]) -> bool:
|
||||
"""Whether a name a Check uses is a setting the resource type lacks.
|
||||
|
||||
A path is judged by the setting at its end, and only when it starts at a name
|
||||
the type has. One that does not start there is a value written with dots, such
|
||||
as a database flag (`cloudsql.iam_authentication`) or a service
|
||||
(`run.googleapis.com`): both were raised as missing settings on 2026-10-09.
|
||||
"""
|
||||
first, _, rest = setting.partition(".")
|
||||
if rest and first not in known:
|
||||
return False
|
||||
return setting.split(".")[-1] not in known
|
||||
|
||||
|
||||
class SchemaReview:
|
||||
def __init__(self, runner: TerraformRunner) -> None:
|
||||
self._runner = runner
|
||||
@@ -111,9 +125,7 @@ class SchemaReview:
|
||||
missing = [
|
||||
setting
|
||||
for setting in named_settings(check.statement)
|
||||
# A resource type named in the statement is not a setting, and a
|
||||
# path is judged by the setting at its end.
|
||||
if setting not in schemas and setting.split(".")[-1] not in known
|
||||
if setting not in schemas and _is_missing(setting, known)
|
||||
]
|
||||
if missing:
|
||||
concerns.append(
|
||||
|
||||
Reference in New Issue
Block a user