"""A lighter model for a first attempt (iteration 5 step 6).""" import dataclasses from langgraph.checkpoint.memory import InMemorySaver from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission from app.adk.roles import load_roles from app.adk.runtime import NoDecider, RoleSpec from app.agent import AGENTS_DIR, ExecutionEngine from app.nodes.router import Router, Thresholds from app.providers import load_provider from app.recorder import InMemoryRecorder from app.workflows.deliverables import DEPENDENCY_GRAPH from app.workflows.deliverables import DeliverableKind as K AUTHOR = DEPENDENCY_GRAPH[K.DEPLOYMENT_GUIDANCE].specialist def roles(**light: str) -> dict[str, RoleSpec]: return { "orchestrator": RoleSpec("orchestrator", "Unused.", budget=40), **{ spec.specialist: RoleSpec( spec.specialist, "Produce.", budget=15, model="strong", first_attempt_model=light.get(spec.specialist.replace("-", "_")), ) for spec in DEPENDENCY_GRAPH.values() }, } class Recording(FakeAgentRuntime): def __init__(self, turns): super().__init__(turns) self.models: list[tuple[str, str | None]] = [] async def run(self, role, task, tools): self.models.append((role.name, role.model)) return await super().run(role, task, tools) def engine(tmp_path, runtime, given) -> ExecutionEngine: return ExecutionEngine( runtime=runtime, roles=given, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), ) def once() -> dict: return { spec.specialist: [[sample_submission(kind)]] for kind, spec in DEPENDENCY_GRAPH.items() } async def test_a_first_attempt_takes_the_lighter_model_and_other_roles_their_own( tmp_path, ): runtime = Recording(once()) await engine(tmp_path, runtime, roles(iac_author="light")).start( "Ingest webhooks", pre_approved=True ) assert dict(runtime.models)[AUTHOR] == "light" assert {model for name, model in runtime.models if name != AUTHOR} == {"strong"} async def test_an_attempt_that_ended_without_a_deliverable_is_retried_on_the_roles_own( tmp_path, ): turns = once() turns[AUTHOR] = [ [Call("needs_input", {"questions": ["Which region?"]})], [sample_submission(K.DEPLOYMENT_GUIDANCE)], ] runtime = Recording(turns) interaction = await engine(tmp_path, runtime, roles(iac_author="light")).start( "Ingest webhooks", pre_approved=True ) assert interaction.status == "completed" assert [model for name, model in runtime.models if name == AUTHOR] == [ "light", "strong", ] def test_the_bundles_lighter_models_reach_roles_only_when_tiers_are_on(): bundle = load_provider("gcp") off = load_roles(AGENTS_DIR, bundle) on = load_roles(AGENTS_DIR, bundle, model_tiers=True) assert all(role.first_attempt_model is None for role in off.values()) assert on[AUTHOR].first_attempt_model == bundle.models.first_attempt[AUTHOR] assert on[AUTHOR].model == bundle.models.for_role(AUTHOR) # The validator is the check on the code; it is never given a lighter model. assert "validator" not in bundle.models.first_attempt assert on == { name: dataclasses.replace( role, first_attempt_model=bundle.models.first_attempt.get(name) ) for name, role in off.items() }