feat: prove the lessons, check overlays in evaluation, and settle the cost steps
Some checks failed
validation / verify (push) Failing after 13s
Some checks failed
validation / verify (push) Failing after 13s
- Schema review: a dotted path is judged only when it starts at a name the resource type has, so a database flag or a service name is no longer read as a missing setting. Lesson L-04, which worked around that, is retired. - Skill routing: in a Change request the question says what changed. - Evaluation: a case may list code_standards the infrastructure code must meet; case 06 checks its overlay's regions, naming and labels. - Model tiers (iteration 5 step 6): a lighter model for a role's first attempt, named in the provider bundle. Measured on case 03 and left off: Flash saved five cents on the IaC author and took twice the requests. - Steps 7 and 9 decided against on the numbers, recorded in the plans. - Plans: the runs of cases 05 and 06, and the first proof of the lessons on case 03 (no Revision, 22 minutes against 28, quality held). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -441,6 +441,32 @@
|
||||
],
|
||||
"overlay_skills": [
|
||||
"example-organisation-standards"
|
||||
],
|
||||
"code_standards": [
|
||||
{
|
||||
"name": "only approved regions",
|
||||
"forbid": "\"(?!europe-west[14]\")(?:us|europe|asia|australia|northamerica|southamerica|me|africa)-[a-z]+[0-9]+\""
|
||||
},
|
||||
{
|
||||
"name": "names carry the environment from a variable",
|
||||
"require": "name\\s*=\\s*\"[^\"]*\\$\\{(?:var|local)\\.[a-z_]*env[a-z_]*\\}"
|
||||
},
|
||||
{
|
||||
"name": "label workload",
|
||||
"require": "\\bworkload\\b\\s*="
|
||||
},
|
||||
{
|
||||
"name": "label environment",
|
||||
"require": "\\benvironment\\b\\s*="
|
||||
},
|
||||
{
|
||||
"name": "label owner",
|
||||
"require": "\\bowner\\b\\s*="
|
||||
},
|
||||
{
|
||||
"name": "label cost-centre",
|
||||
"require": "\"?cost[-_]centre\"?\\s*="
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -71,6 +71,10 @@ class CaseSpec:
|
||||
change: str = ""
|
||||
# Overlay Skills the case's Executions name (ADR-0004).
|
||||
overlay_skills: tuple[str, ...] = ()
|
||||
# What the infrastructure code must and must not contain, as regular
|
||||
# expressions over all of it: how a case checks that standards from an overlay
|
||||
# Skill reached the code. Each is {"name", "require"} or {"name", "forbid"}.
|
||||
code_standards: tuple[Mapping[str, str], ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -114,6 +118,7 @@ def load_cases(path: Path) -> list[CaseSpec]:
|
||||
thresholds=dict(item.get("thresholds", {})),
|
||||
change=str(item.get("change", "")),
|
||||
overlay_skills=tuple(item.get("overlay_skills", ())),
|
||||
code_standards=tuple(item.get("code_standards", ())),
|
||||
)
|
||||
for item in json.loads(path.read_text(encoding="utf-8"))
|
||||
]
|
||||
@@ -295,6 +300,8 @@ async def score_execution(
|
||||
else:
|
||||
terraform_ok, terraform_detail = False, "no Terraform files attached"
|
||||
checks.append(Check("terraform_validates", terraform_ok, terraform_detail))
|
||||
if case.code_standards:
|
||||
checks.append(_code_standards(case.code_standards, terraform_files))
|
||||
|
||||
checks.append(_chain_check(deliverables))
|
||||
checks.append(_verdicts_check(attachments(DeliverableKind.VALIDATION_RESULTS)))
|
||||
@@ -315,6 +322,22 @@ async def score_execution(
|
||||
return CaseResult(case.id, interaction.status, tuple(checks))
|
||||
|
||||
|
||||
def _code_standards(
|
||||
standards: Sequence[Mapping[str, str]], files: Mapping[str, str]
|
||||
) -> Check:
|
||||
"""Whether the code holds what the case requires of it and nothing it forbids."""
|
||||
code = "\n".join(files.values())
|
||||
broken = []
|
||||
for standard in standards:
|
||||
required, forbidden = standard.get("require"), standard.get("forbid")
|
||||
if required and not re.search(required, code):
|
||||
broken.append(f"{standard['name']}: not found")
|
||||
found = re.search(forbidden, code) if forbidden else None
|
||||
if found:
|
||||
broken.append(f"{standard['name']}: found {found.group(0)}")
|
||||
return Check("code_follows_standards", bool(code) and not broken, "; ".join(broken))
|
||||
|
||||
|
||||
def _chain_check(deliverables: Mapping[str, Mapping[str, Any]]) -> Check:
|
||||
"""The Requirements chain holds end to end: every requirement reaches a Check,
|
||||
directly or through a component's Element, and nothing along it is missing."""
|
||||
|
||||
Reference in New Issue
Block a user