feat: prove the lessons, check overlays in evaluation, and settle the cost steps
Some checks failed
validation / verify (push) Failing after 13s

- Schema review: a dotted path is judged only when it starts at a name the
  resource type has, so a database flag or a service name is no longer read
  as a missing setting. Lesson L-04, which worked around that, is retired.
- Skill routing: in a Change request the question says what changed.
- Evaluation: a case may list code_standards the infrastructure code must
  meet; case 06 checks its overlay's regions, naming and labels.
- Model tiers (iteration 5 step 6): a lighter model for a role's first
  attempt, named in the provider bundle. Measured on case 03 and left off:
  Flash saved five cents on the IaC author and took twice the requests.
- Steps 7 and 9 decided against on the numbers, recorded in the plans.
- Plans: the runs of cases 05 and 06, and the first proof of the lessons on
  case 03 (no Revision, 22 minutes against 28, quality held).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 21:27:09 +01:00
parent 567aede5ca
commit e5e1647ff1
35 changed files with 497 additions and 39 deletions

View File

@@ -441,6 +441,32 @@
],
"overlay_skills": [
"example-organisation-standards"
],
"code_standards": [
{
"name": "only approved regions",
"forbid": "\"(?!europe-west[14]\")(?:us|europe|asia|australia|northamerica|southamerica|me|africa)-[a-z]+[0-9]+\""
},
{
"name": "names carry the environment from a variable",
"require": "name\\s*=\\s*\"[^\"]*\\$\\{(?:var|local)\\.[a-z_]*env[a-z_]*\\}"
},
{
"name": "label workload",
"require": "\\bworkload\\b\\s*="
},
{
"name": "label environment",
"require": "\\benvironment\\b\\s*="
},
{
"name": "label owner",
"require": "\\bowner\\b\\s*="
},
{
"name": "label cost-centre",
"require": "\"?cost[-_]centre\"?\\s*="
}
]
}
]

View File

@@ -71,6 +71,10 @@ class CaseSpec:
change: str = ""
# Overlay Skills the case's Executions name (ADR-0004).
overlay_skills: tuple[str, ...] = ()
# What the infrastructure code must and must not contain, as regular
# expressions over all of it: how a case checks that standards from an overlay
# Skill reached the code. Each is {"name", "require"} or {"name", "forbid"}.
code_standards: tuple[Mapping[str, str], ...] = ()
@dataclass(frozen=True)
@@ -114,6 +118,7 @@ def load_cases(path: Path) -> list[CaseSpec]:
thresholds=dict(item.get("thresholds", {})),
change=str(item.get("change", "")),
overlay_skills=tuple(item.get("overlay_skills", ())),
code_standards=tuple(item.get("code_standards", ())),
)
for item in json.loads(path.read_text(encoding="utf-8"))
]
@@ -295,6 +300,8 @@ async def score_execution(
else:
terraform_ok, terraform_detail = False, "no Terraform files attached"
checks.append(Check("terraform_validates", terraform_ok, terraform_detail))
if case.code_standards:
checks.append(_code_standards(case.code_standards, terraform_files))
checks.append(_chain_check(deliverables))
checks.append(_verdicts_check(attachments(DeliverableKind.VALIDATION_RESULTS)))
@@ -315,6 +322,22 @@ async def score_execution(
return CaseResult(case.id, interaction.status, tuple(checks))
def _code_standards(
standards: Sequence[Mapping[str, str]], files: Mapping[str, str]
) -> Check:
"""Whether the code holds what the case requires of it and nothing it forbids."""
code = "\n".join(files.values())
broken = []
for standard in standards:
required, forbidden = standard.get("require"), standard.get("forbid")
if required and not re.search(required, code):
broken.append(f"{standard['name']}: not found")
found = re.search(forbidden, code) if forbidden else None
if found:
broken.append(f"{standard['name']}: found {found.group(0)}")
return Check("code_follows_standards", bool(code) and not broken, "; ".join(broken))
def _chain_check(deliverables: Mapping[str, Mapping[str, Any]]) -> Check:
"""The Requirements chain holds end to end: every requirement reaches a Check,
directly or through a component's Element, and nothing along it is missing."""