Files
gcp_solution_architecture_a…/tests/execution/test_model_tiers.py
Jonathan Boniface e5e1647ff1
Some checks failed
validation / verify (push) Failing after 13s
feat: prove the lessons, check overlays in evaluation, and settle the cost steps
- Schema review: a dotted path is judged only when it starts at a name the
  resource type has, so a database flag or a service name is no longer read
  as a missing setting. Lesson L-04, which worked around that, is retired.
- Skill routing: in a Change request the question says what changed.
- Evaluation: a case may list code_standards the infrastructure code must
  meet; case 06 checks its overlay's regions, naming and labels.
- Model tiers (iteration 5 step 6): a lighter model for a role's first
  attempt, named in the provider bundle. Measured on case 03 and left off:
  Flash saved five cents on the IaC author and took twice the requests.
- Steps 7 and 9 decided against on the numbers, recorded in the plans.
- Plans: the runs of cases 05 and 06, and the first proof of the lessons on
  case 03 (no Revision, 22 minutes against 28, quality held).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 21:27:09 +01:00

115 lines
3.5 KiB
Python

"""A lighter model for a first attempt (iteration 5 step 6)."""
import dataclasses
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.roles import load_roles
from app.adk.runtime import NoDecider, RoleSpec
from app.agent import AGENTS_DIR, ExecutionEngine
from app.nodes.router import Router, Thresholds
from app.providers import load_provider
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH
from app.workflows.deliverables import DeliverableKind as K
AUTHOR = DEPENDENCY_GRAPH[K.DEPLOYMENT_GUIDANCE].specialist
def roles(**light: str) -> dict[str, RoleSpec]:
return {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(
spec.specialist,
"Produce.",
budget=15,
model="strong",
first_attempt_model=light.get(spec.specialist.replace("-", "_")),
)
for spec in DEPENDENCY_GRAPH.values()
},
}
class Recording(FakeAgentRuntime):
def __init__(self, turns):
super().__init__(turns)
self.models: list[tuple[str, str | None]] = []
async def run(self, role, task, tools):
self.models.append((role.name, role.model))
return await super().run(role, task, tools)
def engine(tmp_path, runtime, given) -> ExecutionEngine:
return ExecutionEngine(
runtime=runtime,
roles=given,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
def once() -> dict:
return {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
async def test_a_first_attempt_takes_the_lighter_model_and_other_roles_their_own(
tmp_path,
):
runtime = Recording(once())
await engine(tmp_path, runtime, roles(iac_author="light")).start(
"Ingest webhooks", pre_approved=True
)
assert dict(runtime.models)[AUTHOR] == "light"
assert {model for name, model in runtime.models if name != AUTHOR} == {"strong"}
async def test_an_attempt_that_ended_without_a_deliverable_is_retried_on_the_roles_own(
tmp_path,
):
turns = once()
turns[AUTHOR] = [
[Call("needs_input", {"questions": ["Which region?"]})],
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
]
runtime = Recording(turns)
interaction = await engine(tmp_path, runtime, roles(iac_author="light")).start(
"Ingest webhooks", pre_approved=True
)
assert interaction.status == "completed"
assert [model for name, model in runtime.models if name == AUTHOR] == [
"light",
"strong",
]
def test_the_bundles_lighter_models_reach_roles_only_when_tiers_are_on():
bundle = load_provider("gcp")
off = load_roles(AGENTS_DIR, bundle)
on = load_roles(AGENTS_DIR, bundle, model_tiers=True)
assert all(role.first_attempt_model is None for role in off.values())
assert on[AUTHOR].first_attempt_model == bundle.models.first_attempt[AUTHOR]
assert on[AUTHOR].model == bundle.models.for_role(AUTHOR)
# The validator is the check on the code; it is never given a lighter model.
assert "validator" not in bundle.models.first_attempt
assert on == {
name: dataclasses.replace(
role, first_attempt_model=bundle.models.first_attempt.get(name)
)
for name, role in off.items()
}