feat: prove the lessons, check overlays in evaluation, and settle the cost steps
Some checks failed
validation / verify (push) Failing after 13s
Some checks failed
validation / verify (push) Failing after 13s
- Schema review: a dotted path is judged only when it starts at a name the resource type has, so a database flag or a service name is no longer read as a missing setting. Lesson L-04, which worked around that, is retired. - Skill routing: in a Change request the question says what changed. - Evaluation: a case may list code_standards the infrastructure code must meet; case 06 checks its overlay's regions, naming and labels. - Model tiers (iteration 5 step 6): a lighter model for a role's first attempt, named in the provider bundle. Measured on case 03 and left off: Flash saved five cents on the IaC author and took twice the requests. - Steps 7 and 9 decided against on the numbers, recorded in the plans. - Plans: the runs of cases 05 and 06, and the first proof of the lessons on case 03 (no Revision, 22 minutes against 28, quality held). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
"""Scoring a finished Execution against a benchmark case, without models or network."""
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
@@ -772,3 +773,49 @@ def test_a_functional_requirement_is_reached_through_its_components_elements():
|
||||
|
||||
assert by_name["change_is_recorded"].detail == "FR-1 changed"
|
||||
assert by_name["changed_requirements_reach_checks"].passed
|
||||
|
||||
|
||||
# --- standards an overlay Skill sets reach the code --------------------------------------
|
||||
|
||||
STANDARDS = (
|
||||
{"name": "approved regions", "forbid": r'"us-[a-z]+[0-9]+"'},
|
||||
{"name": "label owner", "require": r"\bowner\b\s*="},
|
||||
)
|
||||
|
||||
|
||||
def code(text: str) -> dict:
|
||||
return {"content": "guide", "attachments": {"terraform/main.tf": text}}
|
||||
|
||||
|
||||
async def test_code_that_follows_the_cases_standards_passes():
|
||||
case = dataclasses.replace(CASE, code_standards=STANDARDS)
|
||||
labelled = code('region = "europe-west1"\nlabels = { owner = var.owner }')
|
||||
|
||||
result = await score_execution(
|
||||
case, execution(deployment_guidance=labelled), HEADINGS, passing_terraform
|
||||
)
|
||||
|
||||
assert check(result, "code_follows_standards").passed
|
||||
|
||||
|
||||
async def test_a_broken_standard_is_named():
|
||||
case = dataclasses.replace(CASE, code_standards=STANDARDS)
|
||||
|
||||
result = await score_execution(
|
||||
case,
|
||||
execution(deployment_guidance=code('region = "us-central1"')),
|
||||
HEADINGS,
|
||||
passing_terraform,
|
||||
)
|
||||
|
||||
broken = check(result, "code_follows_standards")
|
||||
assert not broken.passed
|
||||
assert (
|
||||
broken.detail == 'approved regions: found "us-central1"; label owner: not found'
|
||||
)
|
||||
|
||||
|
||||
async def test_a_case_with_no_standards_has_no_such_check():
|
||||
result = await score_execution(CASE, execution(), HEADINGS, passing_terraform)
|
||||
|
||||
assert all(c.name != "code_follows_standards" for c in result.checks)
|
||||
|
||||
114
tests/execution/test_model_tiers.py
Normal file
114
tests/execution/test_model_tiers.py
Normal file
@@ -0,0 +1,114 @@
|
||||
"""A lighter model for a first attempt (iteration 5 step 6)."""
|
||||
|
||||
import dataclasses
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
|
||||
from app.adk.roles import load_roles
|
||||
from app.adk.runtime import NoDecider, RoleSpec
|
||||
from app.agent import AGENTS_DIR, ExecutionEngine
|
||||
from app.nodes.router import Router, Thresholds
|
||||
from app.providers import load_provider
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH
|
||||
from app.workflows.deliverables import DeliverableKind as K
|
||||
|
||||
AUTHOR = DEPENDENCY_GRAPH[K.DEPLOYMENT_GUIDANCE].specialist
|
||||
|
||||
|
||||
def roles(**light: str) -> dict[str, RoleSpec]:
|
||||
return {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(
|
||||
spec.specialist,
|
||||
"Produce.",
|
||||
budget=15,
|
||||
model="strong",
|
||||
first_attempt_model=light.get(spec.specialist.replace("-", "_")),
|
||||
)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class Recording(FakeAgentRuntime):
|
||||
def __init__(self, turns):
|
||||
super().__init__(turns)
|
||||
self.models: list[tuple[str, str | None]] = []
|
||||
|
||||
async def run(self, role, task, tools):
|
||||
self.models.append((role.name, role.model))
|
||||
return await super().run(role, task, tools)
|
||||
|
||||
|
||||
def engine(tmp_path, runtime, given) -> ExecutionEngine:
|
||||
return ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=given,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
|
||||
|
||||
def once() -> dict:
|
||||
return {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
|
||||
|
||||
async def test_a_first_attempt_takes_the_lighter_model_and_other_roles_their_own(
|
||||
tmp_path,
|
||||
):
|
||||
runtime = Recording(once())
|
||||
|
||||
await engine(tmp_path, runtime, roles(iac_author="light")).start(
|
||||
"Ingest webhooks", pre_approved=True
|
||||
)
|
||||
|
||||
assert dict(runtime.models)[AUTHOR] == "light"
|
||||
assert {model for name, model in runtime.models if name != AUTHOR} == {"strong"}
|
||||
|
||||
|
||||
async def test_an_attempt_that_ended_without_a_deliverable_is_retried_on_the_roles_own(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once()
|
||||
turns[AUTHOR] = [
|
||||
[Call("needs_input", {"questions": ["Which region?"]})],
|
||||
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
||||
]
|
||||
runtime = Recording(turns)
|
||||
|
||||
interaction = await engine(tmp_path, runtime, roles(iac_author="light")).start(
|
||||
"Ingest webhooks", pre_approved=True
|
||||
)
|
||||
|
||||
assert interaction.status == "completed"
|
||||
assert [model for name, model in runtime.models if name == AUTHOR] == [
|
||||
"light",
|
||||
"strong",
|
||||
]
|
||||
|
||||
|
||||
def test_the_bundles_lighter_models_reach_roles_only_when_tiers_are_on():
|
||||
bundle = load_provider("gcp")
|
||||
|
||||
off = load_roles(AGENTS_DIR, bundle)
|
||||
on = load_roles(AGENTS_DIR, bundle, model_tiers=True)
|
||||
|
||||
assert all(role.first_attempt_model is None for role in off.values())
|
||||
assert on[AUTHOR].first_attempt_model == bundle.models.first_attempt[AUTHOR]
|
||||
assert on[AUTHOR].model == bundle.models.for_role(AUTHOR)
|
||||
# The validator is the check on the code; it is never given a lighter model.
|
||||
assert "validator" not in bundle.models.first_attempt
|
||||
assert on == {
|
||||
name: dataclasses.replace(
|
||||
role, first_attempt_model=bundle.models.first_attempt.get(name)
|
||||
)
|
||||
for name, role in off.items()
|
||||
}
|
||||
@@ -240,6 +240,23 @@ async def test_a_setting_found_anywhere_in_the_resource_type_is_accepted():
|
||||
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) == ()
|
||||
|
||||
|
||||
async def test_a_value_written_with_dots_is_not_read_as_a_setting():
|
||||
"""A database flag or a service name is a value. A path is a setting only when
|
||||
it starts at a name the resource type has."""
|
||||
design = graph(
|
||||
[
|
||||
("E-1", "the flag cloudsql.iam_authentication is on"),
|
||||
("E-1", "audit logs are on for run.googleapis.com"),
|
||||
("E-1", "template.scaling.made_up_count is at least 2"),
|
||||
("E-2", "the name is set"),
|
||||
]
|
||||
)
|
||||
|
||||
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
|
||||
|
||||
assert len(raised) == 1 and "template.scaling.made_up_count" in raised[0]
|
||||
|
||||
|
||||
async def test_a_resource_type_the_provider_lacks_is_raised_with_close_matches():
|
||||
design = graph(
|
||||
[("E-1", "the name is set"), ("E-2", "the name is set")],
|
||||
|
||||
Reference in New Issue
Block a user