This commit is contained in:
@@ -271,6 +271,21 @@ async def test_model_timeout_is_set_on_every_request():
|
||||
assert model.requests[0].config.http_options.timeout == 90_000
|
||||
|
||||
|
||||
async def test_a_role_with_its_own_timeout_is_given_it_over_the_default():
|
||||
model = ScriptedLlm(script=[[call("delegate")]])
|
||||
runtime = AdkAgentRuntime(
|
||||
skill_dirs=SKILLS, model_factory=lambda role: model, model_timeout_seconds=90
|
||||
)
|
||||
|
||||
await runtime.run(
|
||||
RoleSpec("orchestrator", "Coordinate.", budget=5, model_timeout_seconds=420),
|
||||
TASK,
|
||||
[Recorder().tool("delegate", TurnEnded("delegate"))],
|
||||
)
|
||||
|
||||
assert model.requests[0].config.http_options.timeout == 420_000
|
||||
|
||||
|
||||
class MeteredLlm(ScriptedLlm):
|
||||
async def generate_content_async(
|
||||
self, llm_request: LlmRequest, stream: bool = False
|
||||
|
||||
@@ -19,7 +19,7 @@ def bundle():
|
||||
def test_every_engine_role_and_the_pillar_advisor_is_defined(bundle):
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
|
||||
expected = {"orchestrator", PILLAR_ADVISOR}
|
||||
expected = {"orchestrator", "reply-writer", PILLAR_ADVISOR}
|
||||
expected |= {spec.specialist for spec in DEPENDENCY_GRAPH.values()}
|
||||
assert set(roles) == expected
|
||||
for role in roles.values():
|
||||
@@ -46,12 +46,35 @@ def test_roles_take_skills_mcp_servers_and_models_from_the_bundle(bundle):
|
||||
def test_roles_name_only_the_local_tools_they_need(bundle):
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
|
||||
assert roles["iac-author"].tools == ("terraform_fmt", "terraform_validate")
|
||||
assert roles["target-architect"].tools == ()
|
||||
assert roles["validator"].tools == ("terraform_validate", "terraform_plan")
|
||||
assert roles["iac-author"].tools == (
|
||||
"terraform_fmt",
|
||||
"terraform_validate",
|
||||
"terraform_schema",
|
||||
)
|
||||
# The architect only looks settings up; it writes and runs no code.
|
||||
assert roles["target-architect"].tools == ("terraform_schema",)
|
||||
assert roles["validator"].tools == (
|
||||
"terraform_validate",
|
||||
"terraform_plan",
|
||||
"terraform_schema",
|
||||
)
|
||||
assert roles["requirements-analyst"].tools == ()
|
||||
|
||||
|
||||
def test_roles_with_long_replies_are_given_longer_for_a_model_request(bundle):
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
|
||||
assert roles["target-architect"].model_timeout_seconds == 420
|
||||
assert roles["decomposer"].model_timeout_seconds is None
|
||||
|
||||
|
||||
def test_the_code_author_is_given_the_patterns_for_its_cloud_provider(bundle):
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
|
||||
assert "google-cloud-iac-patterns" in roles["iac-author"].skills
|
||||
assert "google-cloud-iac-patterns" not in roles["validator"].skills
|
||||
|
||||
|
||||
def test_preview_mcp_servers_can_be_excluded(bundle):
|
||||
roles = load_roles(AGENTS_DIR, bundle, include_preview_mcp=False)
|
||||
|
||||
|
||||
@@ -475,3 +475,56 @@ async def test_a_scripted_user_drives_an_interactive_execution(tmp_path):
|
||||
assert any("Approved." in reply for reply in replies)
|
||||
assert any("europe-west2." in reply for reply in replies)
|
||||
assert final.state["deliverables"]["requirements"]["content"]
|
||||
|
||||
|
||||
# --- Jev as the judge -------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_jev_judges_every_question_in_one_call_with_no_grading_in_the_state():
|
||||
from app.adk.fake_runtime import FakeDecider
|
||||
from app.adk.runtime import Answer
|
||||
from eval.judge import DeciderJudge
|
||||
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"covered": Answer(0.91, 0.82),
|
||||
"missed": Answer(0.2, 0.6),
|
||||
"specific": Answer(2.0, 0.7), # "mostly", on a scale of four
|
||||
}
|
||||
)
|
||||
judge = DeciderJudge(decider)
|
||||
|
||||
scores = await judge.answer(
|
||||
"## Reliability\nUse two zones.",
|
||||
{
|
||||
"covered": {"kind": "yes_no", "text": "Is redundancy addressed?"},
|
||||
"missed": {"kind": "yes_no", "text": "Are backups tested?"},
|
||||
"specific": {"kind": "graded", "text": "Is it specific?"},
|
||||
},
|
||||
)
|
||||
|
||||
assert scores == {"covered": 1.0, "missed": 0.0, "specific": 2 / 3}
|
||||
[(state, questions, scope)] = decider.asked
|
||||
assert state == "## Reliability\nUse two zones." # the document, nothing else
|
||||
assert questions["covered"].kind == "noul"
|
||||
assert questions["specific"].options == ("not at all", "weakly", "mostly", "fully")
|
||||
assert scope == "evaluation"
|
||||
|
||||
|
||||
async def test_a_judge_that_gives_no_answer_leaves_the_check_unanswered_not_passed():
|
||||
from app.adk.runtime import NoDecider
|
||||
from eval.judge import DeciderJudge
|
||||
|
||||
result = await score_execution(
|
||||
CASE,
|
||||
execution(),
|
||||
HEADINGS,
|
||||
passing_terraform,
|
||||
judge=DeciderJudge(NoDecider()),
|
||||
pillar_checklists=CHECKLISTS,
|
||||
)
|
||||
|
||||
assert check(result, "pillar_checklists_covered").detail == (
|
||||
"judge unavailable: Jev gave no answer"
|
||||
)
|
||||
assert not check(result, "recommendations_quality").passed
|
||||
|
||||
@@ -15,7 +15,7 @@ from app.agent import ExecutionEngine
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.tools.local import mermaid_check
|
||||
from app.workflows.chain import validation_outcome
|
||||
from app.workflows.chain import unimplementable_checks, validation_outcome
|
||||
from app.workflows.deliverables import (
|
||||
DEPENDENCY_GRAPH,
|
||||
DeliverableKind,
|
||||
@@ -327,6 +327,54 @@ def test_deployment_guidance_must_account_for_every_check():
|
||||
)
|
||||
|
||||
|
||||
def test_a_check_that_cannot_be_implemented_is_accounted_for_with_its_reason():
|
||||
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
|
||||
implementation = json.dumps(
|
||||
[
|
||||
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
|
||||
{
|
||||
"check": "K-2",
|
||||
"unimplementable": "example_service_account has no such argument",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
missing = check_submission(
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
args["content"],
|
||||
{**args["attachments"], "implementation.json": implementation},
|
||||
upstream(*EVERYTHING_BEFORE_CODE),
|
||||
)
|
||||
|
||||
assert missing == ()
|
||||
assert unimplementable_checks(implementation) == [
|
||||
{"check": "K-2", "reason": "example_service_account has no such argument"}
|
||||
]
|
||||
assert unimplementable_checks(SAMPLE_DATA["implementation.json"]) == []
|
||||
|
||||
|
||||
def test_an_unimplementable_entry_needs_its_reason():
|
||||
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
|
||||
implementation = json.dumps(
|
||||
[
|
||||
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
|
||||
{"check": "K-2", "unimplementable": ""},
|
||||
]
|
||||
)
|
||||
|
||||
missing = check_submission(
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
args["content"],
|
||||
{**args["attachments"], "implementation.json": implementation},
|
||||
upstream(*EVERYTHING_BEFORE_CODE),
|
||||
)
|
||||
|
||||
assert missing == (
|
||||
"implementation.json[0]: needs unimplementable",
|
||||
"implementation.json: Check K-2 is not accounted for",
|
||||
)
|
||||
|
||||
|
||||
def verdicts(**changes) -> str:
|
||||
data = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
data.update(changes)
|
||||
|
||||
598
tests/execution/test_router.py
Normal file
598
tests/execution/test_router.py
Normal file
@@ -0,0 +1,598 @@
|
||||
"""The Orchestrator as a router (ADR-0007): readings, a disposition, then a playbook.
|
||||
|
||||
No language model chooses an action here. Specialists are scripted; the Orchestrator
|
||||
is the Router, with a scripted decider standing in for Jev, or none at all.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import (
|
||||
SAMPLE_DATA,
|
||||
Call,
|
||||
FakeAgentRuntime,
|
||||
FakeDecider,
|
||||
sample_submission,
|
||||
)
|
||||
from app.adk.runtime import Answer, NoDecider, RoleSpec
|
||||
from app.agent import ExecutionEngine
|
||||
from app.nodes.router import (
|
||||
REPLY_WRITER,
|
||||
Router,
|
||||
RuntimeReplyWriter,
|
||||
Thresholds,
|
||||
)
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
||||
|
||||
K = DeliverableKind
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused by the router.", budget=40),
|
||||
REPLY_WRITER: RoleSpec(REPLY_WRITER, "Write the reply.", budget=3),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def specialist(kind: DeliverableKind) -> str:
|
||||
return DEPENDENCY_GRAPH[kind].specialist
|
||||
|
||||
|
||||
def once() -> dict:
|
||||
return {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
|
||||
|
||||
def validation(**changes) -> Call:
|
||||
"""A validation submission whose verdicts differ from the all-pass sample."""
|
||||
call = sample_submission(K.VALIDATION_RESULTS)
|
||||
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
verdicts.update(changes)
|
||||
return Call(
|
||||
call.tool,
|
||||
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
|
||||
)
|
||||
|
||||
|
||||
def failing(check: str = "K-1") -> Call:
|
||||
checks = json.loads(SAMPLE_DATA["verdicts.json"])["checks"]
|
||||
for item in checks:
|
||||
if item["check"] == check:
|
||||
item.update(verdict="fail", evidence="min instances is 1")
|
||||
return validation(checks=checks)
|
||||
|
||||
|
||||
def addition(reason: str) -> Call:
|
||||
return validation(
|
||||
proposed_additions=[
|
||||
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": reason}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def engine_for(tmp_path, turns: dict, decider=None, writer: bool = False, **options):
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
router = Router(
|
||||
decider or NoDecider(),
|
||||
RuntimeReplyWriter(runtime, ROLES[REPLY_WRITER]) if writer else None,
|
||||
Thresholds(),
|
||||
)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=router,
|
||||
**options,
|
||||
)
|
||||
return engine, runtime
|
||||
|
||||
|
||||
def actions(result) -> list[str]:
|
||||
return [decision["action"] for decision in result.state["decisions"]]
|
||||
|
||||
|
||||
def reopened(result) -> list[tuple[str, str]]:
|
||||
return [
|
||||
(event["subject"], event["detail"])
|
||||
for event in result.state["ledger"]["events"]
|
||||
if event["type"] == "reopened"
|
||||
]
|
||||
|
||||
|
||||
def approve(phase: str) -> dict:
|
||||
return {"approval": {"phase": phase, "decision": "approve"}}
|
||||
|
||||
|
||||
# --- the playbook on rules alone -------------------------------------------------------
|
||||
|
||||
|
||||
async def test_an_execution_completes_with_no_model_choosing_an_action(tmp_path):
|
||||
engine, runtime = engine_for(tmp_path, once())
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert "orchestrator" not in {run.role for run in runtime.runs}
|
||||
assert actions(result) == [
|
||||
"delegate", # requirements
|
||||
"delegate", # technical decomposition
|
||||
"request_approval",
|
||||
"delegate", # product selection
|
||||
"delegate", # design recommendations
|
||||
"delegate", # Target architecture
|
||||
"request_approval",
|
||||
"delegate", # description and deployment guidance, together
|
||||
"request_approval",
|
||||
"delegate", # validation
|
||||
"proceed",
|
||||
"request_approval",
|
||||
"delegate", # solution guide
|
||||
"request_approval",
|
||||
"finish",
|
||||
]
|
||||
groups = {decision["group"] for decision in result.state["decisions"]}
|
||||
assert groups == {
|
||||
"work is ready",
|
||||
"a Phase is complete",
|
||||
"validation decided",
|
||||
"packaging is approved",
|
||||
}
|
||||
|
||||
|
||||
async def test_every_decision_is_recorded_with_its_turn_group_and_action(tmp_path):
|
||||
engine, _ = engine_for(tmp_path, once())
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
first = result.state["decisions"][0]
|
||||
assert first == {
|
||||
"turn": 1,
|
||||
"group": "work is ready",
|
||||
"action": "delegate",
|
||||
"judged": {},
|
||||
}
|
||||
assert [d["turn"] for d in result.state["decisions"]] == sorted(
|
||||
d["turn"] for d in result.state["decisions"]
|
||||
)
|
||||
|
||||
|
||||
async def test_when_no_group_applies_a_pre_approved_execution_ends_with_warnings(
|
||||
tmp_path,
|
||||
):
|
||||
turns = {specialist(K.REQUIREMENTS): [[]]} # stops without submitting
|
||||
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
# The Specialist's question is relayed, answered automatically, and it stops
|
||||
# again; the router never invents work.
|
||||
assert result.status == "completed_with_warnings"
|
||||
|
||||
|
||||
# --- Approvals: the summary is written from data, and only data approves ----------------
|
||||
|
||||
|
||||
async def test_the_approval_summary_is_assembled_from_the_data(tmp_path):
|
||||
engine, _ = engine_for(tmp_path, once())
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
design = await engine.resume(first.execution_id, approve("requirements_discovery"))
|
||||
|
||||
assert first.prompt["approval"]["summary"] == (
|
||||
"The requirements_discovery Phase is ready: requirements, "
|
||||
"technical_decomposition submitted. 2 requirements."
|
||||
)
|
||||
summary = design.prompt["approval"]["summary"]
|
||||
assert "Target architecture: 2 Elements, 1 Relationships, 2 Checks." in summary
|
||||
assert design.prompt["approval"]["target_architecture"]["checks"] == 2
|
||||
|
||||
|
||||
async def test_a_model_writes_the_summary_but_decides_nothing(tmp_path):
|
||||
turns = {
|
||||
**once(),
|
||||
REPLY_WRITER: [[Call("write_reply", {"text": "Two requirements, ready."})]],
|
||||
}
|
||||
engine, runtime = engine_for(tmp_path, turns, writer=True)
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
|
||||
assert first.prompt["approval"]["summary"] == "Two requirements, ready."
|
||||
writer_run = next(run for run in runtime.runs if run.role == REPLY_WRITER)
|
||||
assert writer_run.tools == ("write_reply",)
|
||||
assert writer_run.task.context["facts"]["deliverables"] == [
|
||||
"requirements",
|
||||
"technical_decomposition",
|
||||
]
|
||||
|
||||
|
||||
async def test_a_writer_that_fails_does_not_stop_the_approval(tmp_path):
|
||||
engine, _ = engine_for(tmp_path, once(), writer=True) # no writer turn scripted
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
|
||||
assert first.prompt["approval"]["summary"].startswith(
|
||||
"The requirements_discovery Phase is ready"
|
||||
)
|
||||
|
||||
|
||||
async def test_a_text_reply_never_approves_and_the_approval_is_asked_again(tmp_path):
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"asks_change": Answer(0.03, 0.94),
|
||||
"change_target": Answer("requirements", 0.5),
|
||||
"change_clarity": Answer(0.0, 0.9),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, once(), decider)
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
again = await engine.resume(first.execution_id, {"reply": "Looks good to me."})
|
||||
|
||||
assert again.prompt["approval"]["phase"] == "requirements_discovery"
|
||||
assert again.state["ledger"]["phases"]["requirements_discovery"] != "approved"
|
||||
|
||||
|
||||
# --- validation: the code, the graph, or proceed ----------------------------------------
|
||||
|
||||
|
||||
async def test_a_failed_check_sends_the_code_back_and_leaves_the_design_approved(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once()
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [
|
||||
[failing("K-1")],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
engine, runtime = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert reopened(result) == [
|
||||
("deployment_guidance", "K-1 failed: min instances is 1")
|
||||
]
|
||||
assert "revise_code" in actions(result)
|
||||
architect_runs = [
|
||||
run for run in runtime.runs if run.role == specialist(K.TARGET_ARCHITECTURE)
|
||||
]
|
||||
assert len(architect_runs) == 1
|
||||
|
||||
|
||||
async def test_a_check_the_code_cannot_meet_goes_back_to_the_graph_unvalidated(
|
||||
tmp_path,
|
||||
):
|
||||
"""The code's author found the Check names a setting the resource lacks. No
|
||||
validation is spent on it: the Target architecture is revised first."""
|
||||
cannot = sample_submission(K.DEPLOYMENT_GUIDANCE)
|
||||
implementation = json.loads(SAMPLE_DATA["implementation.json"])
|
||||
implementation[1] = {"check": "K-2", "unimplementable": "no such argument"}
|
||||
turns = once()
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [
|
||||
[
|
||||
Call(
|
||||
cannot.tool,
|
||||
{
|
||||
**cannot.args,
|
||||
"attachments": {
|
||||
**cannot.args["attachments"],
|
||||
"implementation.json": json.dumps(implementation),
|
||||
},
|
||||
},
|
||||
)
|
||||
],
|
||||
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
||||
]
|
||||
engine, runtime = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert reopened(result) == [
|
||||
(
|
||||
"target_architecture",
|
||||
(
|
||||
"Check K-2 cannot be implemented as written (no such argument). "
|
||||
"Rewrite it so it can be met, or remove it with a reason."
|
||||
),
|
||||
)
|
||||
]
|
||||
assert "the code cannot meet a Check" in {
|
||||
decision["group"] for decision in result.state["decisions"]
|
||||
}
|
||||
validations = [
|
||||
run for run in runtime.runs if run.role == specialist(K.VALIDATION_RESULTS)
|
||||
]
|
||||
assert len(validations) == 1
|
||||
|
||||
|
||||
async def test_a_check_that_fails_twice_running_is_sent_back_to_the_graph(tmp_path):
|
||||
"""The code was revised and the same Check failed again: the Check is suspect."""
|
||||
turns = once()
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 3
|
||||
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [
|
||||
[failing("K-1")],
|
||||
[failing("K-1")],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
[first, second] = reopened(result)
|
||||
assert first[0] == "deployment_guidance"
|
||||
assert second[0] == "target_architecture"
|
||||
assert "K-1 failed in two validations in a row" in second[1]
|
||||
assert actions(result).count("revise_code") == 1
|
||||
|
||||
|
||||
async def test_without_jev_a_gap_naming_a_real_requirement_revises_the_graph(tmp_path):
|
||||
turns = once()
|
||||
for kind in (
|
||||
K.TARGET_ARCHITECTURE,
|
||||
K.ARCHITECTURE_DESCRIPTION,
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
):
|
||||
turns[specialist(kind)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [
|
||||
[addition("NFR-1 has no Check on timeouts")],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
[(kind, reason)] = reopened(result)
|
||||
assert kind == "target_architecture"
|
||||
assert reason == (
|
||||
"Add a Check on E-1: A timeout is set. (serves NFR-1 has no Check on timeouts)."
|
||||
)
|
||||
|
||||
|
||||
async def test_without_jev_a_gap_that_names_nothing_real_is_left(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("it would be nice")]]
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert reopened(result) == []
|
||||
|
||||
|
||||
async def test_jev_judges_whether_a_gap_is_worth_a_revision(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("NFR-1")]]
|
||||
immaterial = FakeDecider({"gap_material": Answer(0.04, 0.92)})
|
||||
engine, _ = engine_for(tmp_path, turns, immaterial)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
# The rule alone would have revised the graph; the reading says it is not needed.
|
||||
assert reopened(result) == []
|
||||
judged = next(d for d in result.state["decisions"] if d["action"] == "proceed")
|
||||
assert judged["judged"] == {"gap_material": {"value": 0.04, "confidence": 0.92}}
|
||||
[(state, questions, scope)] = immaterial.asked
|
||||
assert "A timeout is set." in state and scope == result.execution_id
|
||||
assert questions["gap_material"].kind == "noul"
|
||||
|
||||
|
||||
async def test_an_uncertain_gap_is_put_to_the_user_who_decides(tmp_path):
|
||||
turns = once()
|
||||
for kind in (
|
||||
K.TARGET_ARCHITECTURE,
|
||||
K.ARCHITECTURE_DESCRIPTION,
|
||||
K.DEPLOYMENT_GUIDANCE,
|
||||
):
|
||||
turns[specialist(kind)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [
|
||||
[addition("NFR-1")],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"gap_material": Answer(0.5, 0.0), # the grey zone
|
||||
"wants_revision": Answer(0.96, 0.92),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
|
||||
result = await engine.start("Ingest webhooks")
|
||||
for phase in (
|
||||
"requirements_discovery",
|
||||
"solution_architecture",
|
||||
"solution_implementation",
|
||||
):
|
||||
result = await engine.resume(result.execution_id, approve(phase))
|
||||
|
||||
[question] = result.prompt["questions"]
|
||||
assert (
|
||||
"found gaps in the Target architecture: A timeout is set."
|
||||
in (question["question"])
|
||||
)
|
||||
|
||||
result = await engine.resume(result.execution_id, {"reply": "Yes, revise it."})
|
||||
|
||||
assert reopened(result)[0][0] == "target_architecture"
|
||||
# Reopening the graph withdrew the design Approval, so it is asked for again.
|
||||
assert result.prompt["approval"]["phase"] == "solution_architecture"
|
||||
|
||||
|
||||
# --- the user's message: act, queue or close --------------------------------------------
|
||||
|
||||
|
||||
def to_design_approval(engine):
|
||||
async def run():
|
||||
first = await engine.start("Ingest webhooks")
|
||||
return await engine.resume(
|
||||
first.execution_id, approve("requirements_discovery")
|
||||
)
|
||||
|
||||
return run()
|
||||
|
||||
|
||||
def changes(phase: str, feedback: str) -> dict:
|
||||
return {"approval": {"phase": phase, "decision": "changes", "feedback": feedback}}
|
||||
|
||||
|
||||
async def test_a_requested_change_is_routed_to_the_deliverable_it_affects(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"change_target": Answer("design_recommendations", 0.88),
|
||||
"change_clarity": Answer(1.8, 0.8),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
|
||||
design = await to_design_approval(engine)
|
||||
result = await engine.resume(
|
||||
design.execution_id,
|
||||
changes("solution_architecture", "Use customer-managed keys on the topic."),
|
||||
)
|
||||
|
||||
assert reopened(result) == [
|
||||
("design_recommendations", "Use customer-managed keys on the topic.")
|
||||
]
|
||||
# Product selection was not touched; the design is asked for again once redone.
|
||||
assert result.state["ledger"]["deliverables"]["product_selection"]["revisions"] == 0
|
||||
assert result.prompt["approval"]["phase"] == "solution_architecture"
|
||||
[(state, questions, _)] = decider.asked
|
||||
assert "The user's message: Use customer-managed keys on the topic." in state
|
||||
assert questions["change_target"].options == (
|
||||
"requirements",
|
||||
"technical_decomposition",
|
||||
"product_selection",
|
||||
"design_recommendations",
|
||||
"target_architecture",
|
||||
)
|
||||
assert "asks_change" not in questions # a structured request is known to be one
|
||||
|
||||
|
||||
async def test_an_unclear_change_is_confirmed_with_the_user_before_anything_moves(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once()
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"change_target": Answer("product_selection", 0.31), # not relied on
|
||||
"change_clarity": Answer(1.5, 0.7),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
|
||||
design = await to_design_approval(engine)
|
||||
asked = await engine.resume(
|
||||
design.execution_id, changes("solution_architecture", "Tighten it up.")
|
||||
)
|
||||
|
||||
assert reopened(asked) == []
|
||||
[question] = asked.prompt["questions"]
|
||||
assert question["question"].startswith("Do you want product_selection changed?")
|
||||
|
||||
result = await engine.resume(
|
||||
asked.execution_id,
|
||||
{"reply": "No, change the target architecture: add a Check on timeouts."},
|
||||
)
|
||||
|
||||
[(kind, reason)] = reopened(result)
|
||||
assert kind == "target_architecture"
|
||||
assert "Tighten it up." in reason and "add a Check on timeouts" in reason
|
||||
|
||||
|
||||
async def test_a_free_text_reply_that_asks_for_a_change_is_acted_on(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.REQUIREMENTS)] = [
|
||||
[sample_submission(K.REQUIREMENTS)],
|
||||
[sample_submission(K.REQUIREMENTS)],
|
||||
]
|
||||
turns[specialist(K.TECHNICAL_DECOMPOSITION)] *= 2
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"asks_change": Answer(0.93, 0.86),
|
||||
"change_target": Answer("requirements", 0.9),
|
||||
"change_clarity": Answer(1.9, 0.8),
|
||||
}
|
||||
)
|
||||
engine, _ = engine_for(tmp_path, turns, decider)
|
||||
|
||||
first = await engine.start("Ingest webhooks")
|
||||
result = await engine.resume(
|
||||
first.execution_id, {"reply": "Retention must be 7 years, not 30 days."}
|
||||
)
|
||||
|
||||
assert reopened(result) == [
|
||||
("requirements", "Retention must be 7 years, not 30 days.")
|
||||
]
|
||||
|
||||
|
||||
async def test_without_jev_a_change_is_acted_on_only_when_the_deliverable_is_named(
|
||||
tmp_path,
|
||||
):
|
||||
turns = once()
|
||||
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
|
||||
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
|
||||
engine, _ = engine_for(tmp_path, turns)
|
||||
|
||||
design = await to_design_approval(engine)
|
||||
unnamed = await engine.resume(
|
||||
design.execution_id, changes("solution_architecture", "Make it cheaper.")
|
||||
)
|
||||
|
||||
assert reopened(unnamed) == []
|
||||
assert unnamed.prompt["questions"][0]["question"].startswith(
|
||||
"Which Deliverable should change"
|
||||
)
|
||||
|
||||
named = await engine.resume(
|
||||
unnamed.execution_id, {"reply": "The design recommendations."}
|
||||
)
|
||||
|
||||
[(kind, reason)] = reopened(named)
|
||||
assert kind == "design_recommendations"
|
||||
assert reason == (
|
||||
"Make it cheaper.\nThe user then said: The design recommendations."
|
||||
)
|
||||
|
||||
|
||||
# --- permission is the Ledger's, whatever was decided ------------------------------------
|
||||
|
||||
|
||||
async def test_past_the_revision_limit_the_user_decides(tmp_path):
|
||||
turns = once()
|
||||
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
|
||||
turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]]
|
||||
engine, _ = engine_for(tmp_path, turns, max_revisions=1)
|
||||
|
||||
result = await engine.start("Ingest webhooks")
|
||||
for phase in (
|
||||
"requirements_discovery",
|
||||
"solution_architecture",
|
||||
"solution_implementation",
|
||||
"solution_implementation",
|
||||
):
|
||||
result = await engine.resume(result.execution_id, approve(phase))
|
||||
|
||||
assert len(reopened(result)) == 1
|
||||
[question] = result.prompt["questions"]
|
||||
assert (
|
||||
"deployment_guidance has reached its revision limit (1)"
|
||||
in (question["question"])
|
||||
)
|
||||
480
tests/execution/test_working_copy.py
Normal file
480
tests/execution/test_working_copy.py
Normal file
@@ -0,0 +1,480 @@
|
||||
"""A Specialist's working copy, and what each Specialist is given to read.
|
||||
|
||||
Three things made an Execution slow: a Specialist rewrote a whole Deliverable in one
|
||||
reply, sent every file again on each tool call, and read every earlier Deliverable in
|
||||
full. A working copy and trimmed reading remove each of them.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import (
|
||||
SAMPLE_DATA,
|
||||
Call,
|
||||
FakeAgentRuntime,
|
||||
sample_submission,
|
||||
)
|
||||
from app.adk.runtime import NoDecider, RoleSpec, Submitted, ToolResult
|
||||
from app.agent import ExecutionEngine
|
||||
from app.nodes.router import Router, Thresholds
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.tools.local import TerraformRunner, local_tools
|
||||
from app.tools.toolkit import make_tool_factory
|
||||
from app.tools.workspace import file_tools, graph_tools
|
||||
from app.workflows.deliverables import (
|
||||
DEPENDENCY_GRAPH,
|
||||
DeliverableKind,
|
||||
dependencies_for,
|
||||
)
|
||||
from app.workflows.target_architecture import parse_graph, render_tables
|
||||
|
||||
K = DeliverableKind
|
||||
GRAPH = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
UPSTREAM = {
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
||||
},
|
||||
"technical_decomposition": {
|
||||
"attachments": {"components.json": SAMPLE_DATA["components.json"]}
|
||||
},
|
||||
"product_selection": {
|
||||
"attachments": {"products.json": SAMPLE_DATA["products.json"]}
|
||||
},
|
||||
"design_recommendations": {
|
||||
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
|
||||
},
|
||||
"target_architecture": {
|
||||
"attachments": {
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def tools_of(kind, working, upstream=UPSTREAM, extra=()):
|
||||
return {
|
||||
tool.name: tool
|
||||
for tool in specialist_tools(
|
||||
kind, extra, allow_cloud_read=False, upstream=upstream, working=working
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
# --- the Target architecture is written once -------------------------------------------
|
||||
|
||||
|
||||
def test_tables_are_rendered_from_the_graph():
|
||||
tables = render_tables(parse_graph(SAMPLE_DATA["target-architecture.json"]))
|
||||
|
||||
assert "## Elements" in tables and "## Relationships" in tables
|
||||
assert (
|
||||
"| E-1 | Ingress service | Example Run | example_run_service | C-1 |" in tables
|
||||
)
|
||||
assert "| R-1 | Ingress service | runs as | Ingress identity |" in tables
|
||||
assert "| K-1 | E-1 | The service runs at least two instances. | NFR-1 |" in tables
|
||||
|
||||
|
||||
def test_a_table_cell_cannot_break_its_row():
|
||||
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
graph["checks"][0]["statement"] = "a | b\nc"
|
||||
|
||||
assert "| a \\| b c |" in render_tables(parse_graph(json.dumps(graph)))
|
||||
|
||||
|
||||
async def test_the_engine_writes_the_tables_and_the_diagram_on_submission():
|
||||
tools = tools_of(K.TARGET_ARCHITECTURE, {})
|
||||
|
||||
result = await tools["submit_deliverable"].handler(
|
||||
content="## Design rationale\nOne service, one identity.\n",
|
||||
attachments={
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
},
|
||||
)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
assert result.outcome.content.startswith("## Design rationale")
|
||||
assert "## Checks" in result.outcome.content
|
||||
assert result.outcome.attachments["architecture.mmd"].startswith("flowchart")
|
||||
|
||||
|
||||
async def test_the_graph_is_built_a_few_items_at_a_time_and_attached_on_submission():
|
||||
working: dict = {}
|
||||
tools = tools_of(K.TARGET_ARCHITECTURE, working)
|
||||
|
||||
await tools["graph_put"].handler(elements=GRAPH["elements"])
|
||||
await tools["graph_put"].handler(relationships=GRAPH["relationships"])
|
||||
put = await tools["graph_put"].handler(checks=GRAPH["checks"])
|
||||
assert put.message == (
|
||||
"Added 2, replaced 0. The graph has 2 Elements, 1 Relationships and 2 Checks."
|
||||
)
|
||||
|
||||
result = await tools["submit_deliverable"].handler(
|
||||
content="## Design rationale\nReasons.\n"
|
||||
)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
submitted = json.loads(result.outcome.attachments["target-architecture.json"])
|
||||
assert [item["id"] for item in submitted["checks"]] == ["K-1", "K-2"]
|
||||
|
||||
|
||||
async def test_a_revision_amends_the_graph_that_was_submitted():
|
||||
working: dict = {}
|
||||
tools = tools_of(K.TARGET_ARCHITECTURE, working)
|
||||
await tools["submit_deliverable"].handler(
|
||||
content="## Design rationale\nReasons.\n",
|
||||
attachments={
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
},
|
||||
)
|
||||
|
||||
# A later attempt, with the working memory the Execution kept.
|
||||
tools = tools_of(K.TARGET_ARCHITECTURE, working)
|
||||
changed = {**GRAPH["checks"][0], "statement": "min_instance_count is at least 2"}
|
||||
put = await tools["graph_put"].handler(checks=[changed])
|
||||
read = await tools["graph_read"].handler()
|
||||
|
||||
assert put.message.startswith("Added 0, replaced 1.")
|
||||
assert "min_instance_count is at least 2" in read.message
|
||||
assert len(json.loads(read.message)["checks"]) == 2
|
||||
|
||||
|
||||
async def test_graph_items_can_be_removed_and_an_item_without_an_id_is_refused():
|
||||
working: dict = {}
|
||||
tools = {tool.name: tool for tool in graph_tools(working)}
|
||||
await tools["graph_put"].handler(
|
||||
elements=GRAPH["elements"],
|
||||
not_applicable=[{"recommendation": "DR-9", "reason": "Single region."}],
|
||||
)
|
||||
|
||||
removed = await tools["graph_remove"].handler(ids=["E-2", "DR-9"])
|
||||
refused = await tools["graph_put"].handler(checks=[{"statement": "no id"}])
|
||||
|
||||
assert removed.message.startswith("Removed 2.")
|
||||
assert refused.message.startswith("Refused") and "checks[0]" in refused.message
|
||||
assert [e["id"] for e in working["graph"]["elements"]] == ["E-1"]
|
||||
|
||||
|
||||
# --- files are written once and changed in place ----------------------------------------
|
||||
|
||||
|
||||
async def test_a_file_is_written_once_and_changed_in_place():
|
||||
working: dict = {}
|
||||
tools = {tool.name: tool for tool in file_tools(working)}
|
||||
|
||||
await tools["write_file"].handler(
|
||||
path="terraform/main.tf", content="a = 1\nb = 1\n"
|
||||
)
|
||||
edited = await tools["edit_file"].handler(
|
||||
path="terraform/main.tf", old="b = 1", new="b = 2"
|
||||
)
|
||||
listing = await tools["read_file"].handler()
|
||||
|
||||
assert edited.message == "Edited terraform/main.tf."
|
||||
assert working["files"] == {"terraform/main.tf": "a = 1\nb = 2\n"}
|
||||
assert "- terraform/main.tf (2 lines)" in listing.message
|
||||
|
||||
|
||||
async def test_an_edit_that_does_not_match_exactly_once_changes_nothing():
|
||||
working: dict = {"files": {"main.tf": "x = 1\nx = 1\n"}}
|
||||
tools = {tool.name: tool for tool in file_tools(working)}
|
||||
|
||||
twice = await tools["edit_file"].handler(path="main.tf", old="x = 1", new="x = 2")
|
||||
never = await tools["edit_file"].handler(path="main.tf", old="y", new="z")
|
||||
absent = await tools["edit_file"].handler(path="other.tf", old="x", new="y")
|
||||
unsafe = await tools["write_file"].handler(path="../main.tf", content="x")
|
||||
|
||||
assert "it matches 2" in twice.message and "it matches 0" in never.message
|
||||
assert absent.message.startswith("Refused") and unsafe.message.startswith("Refused")
|
||||
assert working["files"] == {"main.tf": "x = 1\nx = 1\n"}
|
||||
|
||||
|
||||
async def test_the_working_copy_is_what_is_submitted():
|
||||
working: dict = {}
|
||||
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
|
||||
for path in ("terraform/main.tf", "terraform/variables.tf"):
|
||||
await tools["write_file"].handler(path=path, content=f"# {path}\n")
|
||||
await tools["write_file"].handler(
|
||||
path="implementation.json", content=SAMPLE_DATA["implementation.json"]
|
||||
)
|
||||
content = sample_submission(K.DEPLOYMENT_GUIDANCE).args["content"]
|
||||
|
||||
result = await tools["submit_deliverable"].handler(content=content)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
assert set(result.outcome.attachments) == {
|
||||
"terraform/main.tf",
|
||||
"terraform/variables.tf",
|
||||
"implementation.json",
|
||||
}
|
||||
|
||||
|
||||
async def test_a_file_passed_on_submission_replaces_its_working_copy():
|
||||
working: dict = {"files": {"terraform/main.tf": "old\n"}}
|
||||
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
|
||||
call = sample_submission(K.DEPLOYMENT_GUIDANCE)
|
||||
|
||||
result = await tools["submit_deliverable"].handler(**call.args)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
assert result.outcome.attachments["terraform/main.tf"] != "old\n"
|
||||
assert working["files"]["terraform/main.tf"] != "old\n"
|
||||
|
||||
|
||||
def test_only_a_specialist_that_owns_files_can_write_them():
|
||||
assert "write_file" in tools_of(K.DEPLOYMENT_GUIDANCE, {})
|
||||
assert "write_file" not in tools_of(K.VALIDATION_RESULTS, {})
|
||||
assert "graph_put" in tools_of(K.TARGET_ARCHITECTURE, {})
|
||||
assert "graph_put" not in tools_of(K.DEPLOYMENT_GUIDANCE, {})
|
||||
|
||||
|
||||
# --- Terraform runs on the working copy -------------------------------------------------
|
||||
|
||||
|
||||
class Runner(TerraformRunner):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.validated: list[dict[str, str]] = []
|
||||
|
||||
async def validate(self, files):
|
||||
self.validated.append(dict(files))
|
||||
return ToolResult("terraform validate passed:\n")
|
||||
|
||||
async def formatted(self, files):
|
||||
return {name: content.strip() + "\n" for name, content in files.items()}
|
||||
|
||||
|
||||
async def test_terraform_checks_take_no_files_when_there_is_a_working_copy():
|
||||
runner = Runner()
|
||||
files = {"terraform/main.tf": "x = 1\n", "implementation.json": "[]"}
|
||||
(validate,) = local_tools(["terraform_validate"], runner, files)
|
||||
|
||||
await validate.handler()
|
||||
|
||||
assert validate.parameters["required"] == []
|
||||
assert "pass no files" in validate.description
|
||||
# Only the Terraform files are checked.
|
||||
assert runner.validated == [{"terraform/main.tf": "x = 1\n"}]
|
||||
|
||||
|
||||
async def test_formatting_changes_the_working_copy_and_returns_only_names():
|
||||
files = {"terraform/main.tf": " x = 1 \n"}
|
||||
(fmt,) = local_tools(["terraform_fmt"], Runner(), files)
|
||||
|
||||
result = await fmt.handler()
|
||||
|
||||
assert files == {"terraform/main.tf": "x = 1\n"}
|
||||
assert result.message == (
|
||||
"terraform fmt reformatted 1 file(s) in your working copy: terraform/main.tf."
|
||||
)
|
||||
|
||||
|
||||
async def test_files_passed_anyway_are_written_to_the_working_copy_first():
|
||||
runner = Runner()
|
||||
files: dict[str, str] = {}
|
||||
(validate,) = local_tools(["terraform_validate"], runner, files)
|
||||
|
||||
await validate.handler(files=[{"path": "terraform/main.tf", "content": "y"}])
|
||||
|
||||
assert files == {"terraform/main.tf": "y"}
|
||||
assert runner.validated == [files]
|
||||
|
||||
|
||||
def roles_with(tools_by_role: dict[str, tuple[str, ...]]) -> dict[str, RoleSpec]:
|
||||
return {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(
|
||||
spec.specialist,
|
||||
"Produce.",
|
||||
budget=15,
|
||||
tools=tools_by_role.get(spec.specialist, ()),
|
||||
)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def test_the_validator_checks_the_submitted_code_without_being_sent_it(tmp_path):
|
||||
runner = Runner()
|
||||
roles = roles_with({"validator": ("terraform_validate",)})
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
turns["validator"] = [
|
||||
[Call("terraform_validate", {}), sample_submission(K.VALIDATION_RESULTS)]
|
||||
]
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
tool_factory=make_tool_factory(
|
||||
runtime,
|
||||
roles,
|
||||
{},
|
||||
local_tools=lambda working_copy: local_tools(
|
||||
["terraform_validate"], runner, working_copy
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert runner.validated == [
|
||||
{
|
||||
"terraform/main.tf": "# terraform/main.tf\n",
|
||||
"terraform/variables.tf": "# terraform/variables.tf\n",
|
||||
}
|
||||
]
|
||||
# The code is the code author's: validation attaches its verdicts and nothing else.
|
||||
validation = result.state["deliverables"]["validation_results"]
|
||||
assert set(validation["attachments"]) == {"verdicts.json"}
|
||||
|
||||
|
||||
# --- each Specialist reads what it works from -------------------------------------------
|
||||
|
||||
|
||||
def submitted() -> dict[str, dict]:
|
||||
return {
|
||||
kind.value: {
|
||||
"content": f"# {kind}\nProse.\n",
|
||||
"attachments": {
|
||||
name: SAMPLE_DATA.get(name, f"# {name}\n") for name in spec.attachments
|
||||
},
|
||||
}
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
|
||||
|
||||
def test_the_code_is_written_from_the_graph_and_the_ids_alone():
|
||||
given = dependencies_for(K.DEPLOYMENT_GUIDANCE, submitted())
|
||||
|
||||
assert given["target_architecture"] == {
|
||||
"attachments": {
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
}
|
||||
}
|
||||
assert all("content" not in deliverable for deliverable in given.values())
|
||||
assert set(given["requirements"]["attachments"]) == {"requirements.json"}
|
||||
|
||||
|
||||
def test_a_direct_dependency_is_read_whole_and_the_rest_as_data():
|
||||
given = dependencies_for(K.TARGET_ARCHITECTURE, submitted())
|
||||
|
||||
assert "content" in given["product_selection"]
|
||||
assert "content" in given["design_recommendations"]
|
||||
assert "content" not in given["requirements"]
|
||||
assert "content" not in given["technical_decomposition"]
|
||||
|
||||
|
||||
def test_the_validator_reads_the_code_whole_and_the_graph_as_data():
|
||||
given = dependencies_for(K.VALIDATION_RESULTS, submitted())
|
||||
|
||||
assert "terraform/main.tf" in given["deployment_guidance"]["attachments"]
|
||||
assert "content" not in given["target_architecture"]
|
||||
# A Deliverable with no data file has nothing to trim to, so it comes whole.
|
||||
assert "content" in given["architecture_description"]
|
||||
|
||||
|
||||
def test_the_solution_guide_is_assembled_from_every_deliverable_whole():
|
||||
given = dependencies_for(K.SOLUTION_GUIDE, submitted())
|
||||
|
||||
assert len(given) == len(DEPENDENCY_GRAPH) - 1
|
||||
assert all("content" in deliverable for deliverable in given.values())
|
||||
|
||||
|
||||
async def test_a_specialist_is_told_what_its_working_copy_already_holds(tmp_path):
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
failing = sample_submission(K.VALIDATION_RESULTS)
|
||||
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
verdicts["checks"][0].update(verdict="fail", evidence="min instances is 1")
|
||||
turns["validator"] = [
|
||||
[
|
||||
Call(
|
||||
failing.tool,
|
||||
{
|
||||
**failing.args,
|
||||
"attachments": {"verdicts.json": json.dumps(verdicts)},
|
||||
},
|
||||
)
|
||||
],
|
||||
[sample_submission(K.VALIDATION_RESULTS)],
|
||||
]
|
||||
turns["iac-author"] = [[sample_submission(K.DEPLOYMENT_GUIDANCE)]] * 2
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=roles_with({}),
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
|
||||
await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
first, second = (run for run in runtime.runs if run.role == "iac-author")
|
||||
assert first.task.context["working_copy"] is None
|
||||
assert second.task.context["working_copy"] == {
|
||||
"files": ["implementation.json", "terraform/main.tf", "terraform/variables.tf"]
|
||||
}
|
||||
|
||||
|
||||
async def test_a_deliverable_given_as_data_can_be_read_whole_on_demand():
|
||||
library = {
|
||||
"requirements": {
|
||||
"content": "# requirements\nProse.\n",
|
||||
"attachments": {"requirements.json": "[]"},
|
||||
}
|
||||
}
|
||||
tools = {
|
||||
tool.name: tool
|
||||
for tool in specialist_tools(
|
||||
K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False, library=library
|
||||
)
|
||||
}
|
||||
|
||||
read = await tools["read_deliverable"].handler(kind="requirements")
|
||||
refused = await tools["read_deliverable"].handler(kind="solution_guide")
|
||||
|
||||
assert (
|
||||
read.message.startswith("# requirements")
|
||||
and "requirements.json" in read.message
|
||||
)
|
||||
assert refused.message.startswith("Refused")
|
||||
# With nothing trimmed there is nothing to read back.
|
||||
assert "read_deliverable" not in tools_of(K.REQUIREMENTS, {})
|
||||
|
||||
|
||||
async def test_the_engine_offers_the_trimmed_deliverables_for_reading(tmp_path):
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=roles_with({}),
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
|
||||
await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
author = next(run for run in runtime.runs if run.role == "iac-author")
|
||||
packager = next(run for run in runtime.runs if run.role == "packager")
|
||||
assert "read_deliverable" in author.tools
|
||||
assert "content" not in author.task.context["dependencies"]["requirements"]
|
||||
assert "read_deliverable" not in packager.tools
|
||||
@@ -12,6 +12,7 @@ def settings(tmp_path, **overrides) -> Settings:
|
||||
values: dict[str, Any] = {
|
||||
"CHECKPOINTER": "memory",
|
||||
"MCP_ENABLED": False,
|
||||
"ORCHESTRATOR": "model",
|
||||
"ORCHESTRATOR_BUDGET": 9,
|
||||
"SPECIALIST_BUDGET": 4,
|
||||
}
|
||||
@@ -50,3 +51,42 @@ async def test_preview_mcp_servers_follow_the_setting(tmp_path):
|
||||
|
||||
assert "asset_inventory" not in servers
|
||||
assert "compute" in servers
|
||||
|
||||
|
||||
async def test_the_assembly_routes_by_default_and_runs_without_a_jev_key(tmp_path):
|
||||
"""With no Jev key the router follows its playbook on rules alone."""
|
||||
from app.adk.fake_runtime import sample_submission
|
||||
from app.adk.runtime import NoDecider
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
runtime = FakeAgentRuntime(
|
||||
{"requirements-analyst": [[sample_submission(DeliverableKind.REQUIREMENTS)]]}
|
||||
)
|
||||
|
||||
async with open_assembly(
|
||||
settings(tmp_path, ORCHESTRATOR="router", TYPESAFE_API_KEY=None),
|
||||
runtime=runtime,
|
||||
recorder=InMemoryRecorder(),
|
||||
output_root=tmp_path,
|
||||
) as assembly:
|
||||
interaction = await assembly.engine.start("Design ingestion")
|
||||
|
||||
assert isinstance(assembly.decider, NoDecider)
|
||||
# The router delegated requirements with no Orchestrator model, then the
|
||||
# decomposer (unscripted here) stopped and its question reached the user.
|
||||
assert [run.role for run in runtime.runs] == ["requirements-analyst"]
|
||||
assert interaction.status == "awaiting_user"
|
||||
assert "reply-writer" in assembly.roles
|
||||
|
||||
|
||||
async def test_an_unknown_orchestrator_setting_is_refused(tmp_path):
|
||||
import pytest
|
||||
|
||||
with pytest.raises(ValueError, match="ORCHESTRATOR must be router or model"):
|
||||
async with open_assembly(
|
||||
settings(tmp_path, ORCHESTRATOR="llm"),
|
||||
runtime=FakeAgentRuntime({}),
|
||||
recorder=InMemoryRecorder(),
|
||||
output_root=tmp_path,
|
||||
):
|
||||
pass
|
||||
|
||||
62
tests/test_iac_patterns.py
Normal file
62
tests/test_iac_patterns.py
Normal file
@@ -0,0 +1,62 @@
|
||||
"""The IaC patterns Skill: every pattern validates against the pinned provider.
|
||||
|
||||
A pattern that does not validate teaches the mistake it was written to prevent, so
|
||||
each reference's Terraform is run through `terraform validate` with the Skill's own
|
||||
versions file. The provider comes from the plugin cache; with no cache and no
|
||||
network the test is skipped, not failed.
|
||||
"""
|
||||
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import Settings
|
||||
from app.providers import load_provider
|
||||
from app.tools.local import TerraformRunner
|
||||
|
||||
SKILL = "google-cloud-iac-patterns"
|
||||
ROOT = Path(__file__).resolve().parent.parent / "skills" / SKILL
|
||||
REFERENCES = sorted((ROOT / "references").glob("*.md"))
|
||||
_HCL = re.compile(r"^```hcl\n(.*?)^```", re.MULTILINE | re.DOTALL)
|
||||
|
||||
needs_terraform = pytest.mark.skipif(
|
||||
shutil.which("terraform") is None, reason="terraform CLI not installed"
|
||||
)
|
||||
|
||||
|
||||
def _runner() -> TerraformRunner:
|
||||
return TerraformRunner(plugin_cache_dir=Settings().terraform_plugin_cache())
|
||||
|
||||
|
||||
def test_every_reference_is_listed_in_the_skill_and_holds_a_pattern():
|
||||
skill = (ROOT / "SKILL.md").read_text(encoding="utf-8")
|
||||
assert REFERENCES
|
||||
for reference in REFERENCES:
|
||||
assert f"references/{reference.name}" in skill
|
||||
assert _HCL.findall(reference.read_text(encoding="utf-8")), reference.name
|
||||
listed = set(re.findall(r"references/([\w-]+\.md)", skill))
|
||||
assert listed == {reference.name for reference in REFERENCES}
|
||||
|
||||
|
||||
def test_the_skill_pins_the_version_the_provider_bundle_pins():
|
||||
pin = load_provider("gcp").terraform_providers["google"]
|
||||
versions = (ROOT / "assets" / "versions.tf").read_text(encoding="utf-8")
|
||||
skill = (ROOT / "SKILL.md").read_text(encoding="utf-8")
|
||||
assert f'source = "{pin["source"]}"' in versions
|
||||
assert f'version = "{pin["version"]}"' in versions
|
||||
assert f"{pin['source']} {pin['version']}" in skill
|
||||
|
||||
|
||||
@needs_terraform
|
||||
@pytest.mark.parametrize("reference", REFERENCES, ids=lambda path: path.stem)
|
||||
async def test_pattern_validates(reference: Path):
|
||||
files = {
|
||||
"terraform/versions.tf": (ROOT / "assets" / "versions.tf").read_text(),
|
||||
"terraform/main.tf": "\n".join(_HCL.findall(reference.read_text())),
|
||||
}
|
||||
result = await _runner().validate(files)
|
||||
if result.message.startswith("terraform init failed"):
|
||||
pytest.skip("the pinned provider could not be installed")
|
||||
assert result.message.startswith("terraform validate passed"), result.message
|
||||
@@ -218,3 +218,55 @@ def test_bundle_places_each_local_tool_in_its_own_tier():
|
||||
for tier, names in bundle.tools.items():
|
||||
for name in names:
|
||||
assert LOCAL_TOOLS[name][0] == tier, name
|
||||
|
||||
|
||||
def test_bundle_pins_terraform_providers_and_role_timeouts(tmp_path):
|
||||
config = minimal_bundle(
|
||||
tmp_path,
|
||||
models={"default": "gemini/m", "timeout_seconds": {"target-architect": 420}},
|
||||
terraform={
|
||||
"required_providers": {
|
||||
"cloud": {"source": "example/cloud", "version": "1.2.3"}
|
||||
}
|
||||
},
|
||||
)
|
||||
write_bundle(tmp_path, config)
|
||||
|
||||
bundle = load_provider("test", root=tmp_path)
|
||||
|
||||
assert bundle.terraform_providers == {
|
||||
"cloud": {"source": "example/cloud", "version": "1.2.3"}
|
||||
}
|
||||
assert bundle.models.timeout_for("target-architect") == 420
|
||||
assert bundle.models.timeout_for("decomposer") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, message",
|
||||
[
|
||||
(
|
||||
{"terraform": {"required_providers": {"cloud": {"source": "x/cloud"}}}},
|
||||
"needs a 'source' and an exact 'version'",
|
||||
),
|
||||
(
|
||||
{"models": {"default": "gemini/m", "timeout_seconds": {"x": 0}}},
|
||||
"must be positive",
|
||||
),
|
||||
(
|
||||
{"models": {"default": "gemini/m", "timeout_seconds": {"x": "slow"}}},
|
||||
"must be a number",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_bad_terraform_pins_and_timeouts_are_rejected(tmp_path, overrides, message):
|
||||
write_bundle(tmp_path, minimal_bundle(tmp_path, **overrides))
|
||||
|
||||
with pytest.raises(ProviderConfigError, match=message):
|
||||
load_provider("test", root=tmp_path)
|
||||
|
||||
|
||||
def test_gcp_bundle_pins_one_exact_google_provider_version():
|
||||
pin = load_provider("gcp").terraform_providers["google"]
|
||||
|
||||
assert pin["source"] == "hashicorp/google"
|
||||
assert pin["version"].replace(".", "").isdigit() # exact, not a range
|
||||
|
||||
417
tests/tools/test_jev.py
Normal file
417
tests/tools/test_jev.py
Normal file
@@ -0,0 +1,417 @@
|
||||
"""The Jev decision client: typed answers within strict limits, and a fallback always.
|
||||
|
||||
Every test runs the real TypeSafe SDK against a stand-in transport; nothing here calls
|
||||
the live service.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
import httpx2
|
||||
|
||||
from app.adk.runtime import Answer, NoDecider, Question
|
||||
from app.agent import open_decider
|
||||
from app.config import Settings
|
||||
from app.tools.jev import JevDecider, JevLimits, estimate_tokens
|
||||
|
||||
URGENT = Question("noul", "Does this need attention right now?")
|
||||
ROUTE = Question(
|
||||
"choice", "What comes next?", ("revise_code", "revise_graph", "proceed")
|
||||
)
|
||||
SEVERITY = Question("score", "How much does it matter?", ("low", "medium", "high"))
|
||||
ALL = {"urgent": URGENT, "route": ROUTE, "severity": SEVERITY}
|
||||
|
||||
ANSWERS = {
|
||||
"urgent": {"type": "noul", "noul": 0.97},
|
||||
"route": {
|
||||
"type": "choice",
|
||||
"choice": "revise_code",
|
||||
"confidence": 0.81,
|
||||
"probabilities": {"revise_code": 0.81, "revise_graph": 0.12, "proceed": 0.07},
|
||||
},
|
||||
"severity": {
|
||||
"type": "score",
|
||||
"score": 1.6,
|
||||
"confidence": 0.7,
|
||||
"legend": {"0": "low", "1": "medium", "2": "high"},
|
||||
"probabilities": {"0": 0.1, "1": 0.2, "2": 0.7},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class Clock:
|
||||
"""Time that moves only when something sleeps."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.now = 1000.0
|
||||
self.slept: list[float] = []
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.now
|
||||
|
||||
async def sleep(self, seconds: float) -> None:
|
||||
self.slept.append(seconds)
|
||||
self.now += seconds
|
||||
|
||||
|
||||
def ok(*names: str, tokens: int = 40) -> httpx2.Response:
|
||||
answers = {name: ANSWERS[name] for name in names} if names else ANSWERS
|
||||
return httpx2.Response(
|
||||
200,
|
||||
json={
|
||||
"model": "jev-latest",
|
||||
"answers": answers,
|
||||
"usage": {"input_tokens": tokens, "output_tokens": 0},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class Service:
|
||||
"""A stand-in Jev: replies from a queue, then with `default`; records requests.
|
||||
|
||||
A reply answers only the questions the request asked, as the service does.
|
||||
"""
|
||||
|
||||
def __init__(self, *replies, default=None) -> None:
|
||||
self.replies = list(replies)
|
||||
self.default = default
|
||||
self.requests: list[httpx2.Request] = []
|
||||
|
||||
def __call__(self, request: httpx2.Request) -> httpx2.Response:
|
||||
self.requests.append(request)
|
||||
reply = self.replies.pop(0) if self.replies else self.default
|
||||
if reply is None:
|
||||
return ok(*json.loads(request.content)["questions"])
|
||||
if isinstance(reply, Exception):
|
||||
raise reply
|
||||
return reply
|
||||
|
||||
@property
|
||||
def bodies(self) -> list[dict]:
|
||||
return [json.loads(request.content) for request in self.requests]
|
||||
|
||||
|
||||
def decider(service: Service, clock: Clock | None = None, **limits) -> JevDecider:
|
||||
clock = clock or Clock()
|
||||
limits.setdefault("backoff_base", 0.0) # no real waiting between SDK retries
|
||||
return JevDecider(
|
||||
api_key="tsk_secret",
|
||||
base_url="https://jev.example",
|
||||
limits=JevLimits(**limits),
|
||||
transport=httpx2.MockTransport(service),
|
||||
clock=clock,
|
||||
sleep=clock.sleep,
|
||||
)
|
||||
|
||||
|
||||
# --- the request and the typed answers -------------------------------------------------
|
||||
|
||||
|
||||
async def test_one_request_carries_the_state_and_every_question():
|
||||
service = Service()
|
||||
jev = decider(service)
|
||||
|
||||
answers = await jev.decide("K-30 failed twice.", ALL, scope="exec-1")
|
||||
|
||||
[request] = service.requests
|
||||
assert request.method == "POST"
|
||||
assert str(request.url) == "https://jev.example/v1/systemone"
|
||||
assert request.headers["Authorization"] == "Bearer tsk_secret"
|
||||
body = service.bodies[0]
|
||||
assert body["state"] == "K-30 failed twice."
|
||||
assert body["questions"] == {
|
||||
"urgent": {"type": "noul", "instructions": URGENT.instructions},
|
||||
"route": {
|
||||
"type": "choice",
|
||||
"instructions": ROUTE.instructions,
|
||||
"criteria": {"revise_code": None, "revise_graph": None, "proceed": None},
|
||||
},
|
||||
"severity": {
|
||||
"type": "score",
|
||||
"instructions": SEVERITY.instructions,
|
||||
"criteria": ["low", "medium", "high"],
|
||||
},
|
||||
}
|
||||
assert answers["route"] == Answer(
|
||||
"revise_code",
|
||||
0.81,
|
||||
{"revise_code": 0.81, "revise_graph": 0.12, "proceed": 0.07},
|
||||
)
|
||||
assert answers["severity"] == Answer(1.6, 0.7, {"0": 0.1, "1": 0.2, "2": 0.7})
|
||||
# A yes-or-no answer is a probability; its confidence is its distance from even.
|
||||
assert answers["urgent"].value == 0.97
|
||||
assert round(answers["urgent"].confidence, 2) == 0.94
|
||||
assert answers["urgent"].is_yes() and not Answer(0.2, 0.6).is_yes()
|
||||
|
||||
|
||||
async def test_a_choice_outside_the_options_or_a_missing_answer_is_no_decision():
|
||||
invented = httpx2.Response(
|
||||
200,
|
||||
json={
|
||||
"model": "jev-latest",
|
||||
"usage": {"input_tokens": 1, "output_tokens": 0},
|
||||
"answers": {
|
||||
"route": {
|
||||
"type": "choice",
|
||||
"choice": "delete_everything",
|
||||
"confidence": 0.99,
|
||||
"probabilities": {},
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert await decider(Service(invented)).decide("s", {"route": ROUTE}) is None
|
||||
assert await decider(Service(ok("urgent"))).decide("s", ALL) is None
|
||||
assert (
|
||||
await decider(Service(httpx2.Response(200, text="not json"))).decide(
|
||||
"s", {"urgent": URGENT}
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
async def test_no_questions_needs_no_request():
|
||||
service = Service()
|
||||
|
||||
assert await decider(service).decide("state", {}) == {}
|
||||
assert service.requests == []
|
||||
|
||||
|
||||
# --- size --------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_request_over_the_size_limit_is_refused_locally_never_truncated():
|
||||
service = Service()
|
||||
jev = decider(service, max_tokens=100, max_question_tokens=100)
|
||||
state = "x" * 600
|
||||
|
||||
assert estimate_tokens(state) > 100
|
||||
assert await jev.decide(state, {"urgent": URGENT}) is None
|
||||
assert service.requests == []
|
||||
assert jev.records[-1].outcome == "too_large"
|
||||
|
||||
|
||||
# --- timeouts and retries (the SDK's, configured here) ------------------------------------
|
||||
|
||||
|
||||
async def test_temporary_failures_are_retried_then_answered():
|
||||
service = Service(httpx2.Response(503), httpx2.ConnectTimeout("slow"))
|
||||
jev = decider(service)
|
||||
|
||||
assert await jev.decide("s", {"urgent": URGENT}) is not None
|
||||
assert len(service.requests) == 3
|
||||
|
||||
|
||||
async def test_a_rate_limit_that_outlasts_the_retries_lowers_the_clients_own_rate():
|
||||
service = Service(default=httpx2.Response(429))
|
||||
jev = decider(service, retries=1)
|
||||
|
||||
assert await jev.decide("s", {"urgent": URGENT}) is None
|
||||
assert len(service.requests) == 2
|
||||
assert jev.records[-1].rate_limited
|
||||
assert jev._rate == JevLimits().requests_per_second / 2
|
||||
|
||||
|
||||
async def test_a_rejected_request_is_not_retried():
|
||||
service = Service(default=httpx2.Response(400, json={"error": "bad question"}))
|
||||
jev = decider(service)
|
||||
|
||||
assert await jev.decide("s", {"urgent": URGENT}) is None
|
||||
assert len(service.requests) == 1
|
||||
|
||||
|
||||
async def test_retries_stop_at_the_budget_and_the_caller_falls_back():
|
||||
service = Service(default=httpx2.Response(503))
|
||||
jev = decider(service, retries=2)
|
||||
|
||||
assert await jev.decide("s", {"urgent": URGENT}) is None
|
||||
assert len(service.requests) == 3
|
||||
assert jev.records[-1].outcome == "failed"
|
||||
|
||||
|
||||
# --- rate and concurrency ----------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_calls_beyond_the_burst_wait_for_the_bucket_to_refill():
|
||||
clock = Clock()
|
||||
service = Service()
|
||||
jev = decider(service, clock, burst=2, requests_per_second=4)
|
||||
|
||||
for index in range(3):
|
||||
await jev.decide(f"state {index}", {"urgent": URGENT})
|
||||
|
||||
assert len(service.requests) == 3
|
||||
assert clock.slept == [0.25] # one token at four a second
|
||||
|
||||
|
||||
async def test_calls_in_flight_never_exceed_the_cap():
|
||||
in_flight = peak = 0
|
||||
|
||||
async def slow(request: httpx2.Request) -> httpx2.Response:
|
||||
nonlocal in_flight, peak
|
||||
in_flight += 1
|
||||
peak = max(peak, in_flight)
|
||||
await asyncio.sleep(0.01)
|
||||
in_flight -= 1
|
||||
return ok("urgent")
|
||||
|
||||
jev = JevDecider(
|
||||
api_key="k",
|
||||
base_url="https://jev.example",
|
||||
limits=JevLimits(max_in_flight=2, burst=50),
|
||||
transport=httpx2.MockTransport(slow),
|
||||
)
|
||||
|
||||
await asyncio.gather(
|
||||
*(jev.decide(f"state {index}", {"urgent": URGENT}) for index in range(8))
|
||||
)
|
||||
|
||||
assert peak == 2
|
||||
|
||||
|
||||
# --- circuit breaker ---------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_repeated_failures_open_the_circuit_and_one_probe_closes_it():
|
||||
clock = Clock()
|
||||
service = Service(default=httpx2.Response(503))
|
||||
jev = decider(
|
||||
service, clock, retries=0, breaker_failures=3, breaker_cooldown=60, burst=50
|
||||
)
|
||||
|
||||
for index in range(3):
|
||||
assert await jev.decide(f"s{index}", {"urgent": URGENT}) is None
|
||||
sent = len(service.requests)
|
||||
|
||||
# Open: the service is not called at all.
|
||||
assert await jev.decide("s-open", {"urgent": URGENT}) is None
|
||||
assert len(service.requests) == sent
|
||||
assert jev.records[-1].outcome == "breaker_open"
|
||||
|
||||
# After the cool-down a single call probes; its success closes the circuit.
|
||||
clock.now += 61
|
||||
service.default = None
|
||||
assert await jev.decide("s-probe", {"urgent": URGENT}) is not None
|
||||
assert await jev.decide("s-after", {"urgent": URGENT}) is not None
|
||||
|
||||
|
||||
async def test_a_failed_probe_reopens_the_circuit():
|
||||
clock = Clock()
|
||||
service = Service(default=httpx2.Response(503))
|
||||
jev = decider(
|
||||
service, clock, retries=0, breaker_failures=2, breaker_cooldown=60, burst=50
|
||||
)
|
||||
for index in range(2):
|
||||
await jev.decide(f"s{index}", {"urgent": URGENT})
|
||||
|
||||
clock.now += 61
|
||||
await jev.decide("probe", {"urgent": URGENT})
|
||||
sent = len(service.requests)
|
||||
|
||||
assert await jev.decide("again", {"urgent": URGENT}) is None
|
||||
assert len(service.requests) == sent
|
||||
|
||||
|
||||
# --- cache and budget --------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_the_same_question_about_the_same_state_is_answered_from_memory():
|
||||
service = Service()
|
||||
jev = decider(service)
|
||||
|
||||
first = await jev.decide("unchanged", {"route": ROUTE})
|
||||
second = await jev.decide("unchanged", {"route": ROUTE})
|
||||
await jev.decide("changed", {"route": ROUTE})
|
||||
|
||||
assert first == second
|
||||
assert len(service.requests) == 2
|
||||
assert [r.outcome for r in jev.records] == ["answered", "cached", "answered"]
|
||||
|
||||
|
||||
async def test_each_execution_has_a_budget_of_calls_and_then_falls_back(caplog):
|
||||
service = Service()
|
||||
jev = decider(service, budget_calls=2, burst=50)
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
results = [
|
||||
await jev.decide(f"state {index}", {"urgent": URGENT}, scope="exec-1")
|
||||
for index in range(4)
|
||||
]
|
||||
other = await jev.decide("state", {"urgent": URGENT}, scope="exec-2")
|
||||
|
||||
assert [result is not None for result in results] == [True, True, False, False]
|
||||
assert other is not None # another Execution has its own budget
|
||||
assert len(service.requests) == 3
|
||||
assert caplog.text.count("budget reached for exec-1") == 1
|
||||
|
||||
|
||||
async def test_each_execution_has_a_budget_of_tokens():
|
||||
service = Service(default=ok("urgent", tokens=600))
|
||||
jev = decider(service, budget_tokens=1000, burst=50)
|
||||
|
||||
results = [
|
||||
await jev.decide(f"state {index}", {"urgent": URGENT}, scope="exec-1")
|
||||
for index in range(3)
|
||||
]
|
||||
|
||||
assert [result is not None for result in results] == [True, True, False]
|
||||
|
||||
|
||||
async def test_the_key_is_never_logged(caplog):
|
||||
service = Service(httpx2.Response(500), httpx2.Response(401))
|
||||
jev = decider(service, retries=0)
|
||||
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
await jev.decide("a", {"urgent": URGENT})
|
||||
await jev.decide("b", {"urgent": URGENT})
|
||||
|
||||
assert "tsk_secret" not in caplog.text
|
||||
|
||||
|
||||
# --- what is recorded, and running without Jev -------------------------------------------
|
||||
|
||||
|
||||
async def test_every_decision_is_recorded_for_the_evaluation_report():
|
||||
service = Service(ok("urgent"), default=httpx2.Response(400))
|
||||
jev = decider(service)
|
||||
|
||||
await jev.decide("first", {"urgent": URGENT}, scope="exec-1")
|
||||
await jev.decide("first", {"urgent": URGENT}, scope="exec-1")
|
||||
await jev.decide("second", {"urgent": URGENT}, scope="exec-1")
|
||||
|
||||
assert round(jev.records[0].confidence["urgent"], 2) == 0.94
|
||||
assert jev.summary() == {
|
||||
"decisions": 3,
|
||||
"outcomes": {"answered": 1, "cached": 1, "failed": 1},
|
||||
"fallbacks": 1,
|
||||
"rate_limited": 0,
|
||||
"seconds": round(jev.records[0].seconds, 3),
|
||||
"input_tokens": 40,
|
||||
}
|
||||
|
||||
|
||||
async def test_without_a_key_every_caller_takes_its_fallback():
|
||||
unset = open_decider(Settings(_env_file=None, TYPESAFE_API_KEY=None))
|
||||
|
||||
assert isinstance(unset, NoDecider)
|
||||
assert await unset.decide("state", {"urgent": URGENT}) is None
|
||||
|
||||
|
||||
async def test_with_a_key_the_client_takes_its_limits_from_settings():
|
||||
configured = open_decider(
|
||||
Settings(
|
||||
_env_file=None,
|
||||
TYPESAFE_API_KEY="tsk_x",
|
||||
TYPESAFE_BASE_URL="https://jev.internal",
|
||||
JEV_REQUESTS_PER_SECOND=2,
|
||||
JEV_BUDGET_CALLS=40,
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(configured, JevDecider)
|
||||
assert configured.limits.requests_per_second == 2
|
||||
assert configured.limits.budget_calls == 40
|
||||
assert configured.limits.max_tokens == 48_000
|
||||
await configured.aclose()
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Local tools: Mermaid check and Terraform fmt/validate/plan in a scratch directory."""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
@@ -140,6 +141,110 @@ async def test_plan_never_runs_apply_and_refuses_unsafe_variables(monkeypatch):
|
||||
assert "-var=region=europe-west2" in plan
|
||||
|
||||
|
||||
# --- schema lookup -----------------------------------------------------------------------
|
||||
|
||||
SCHEMA = {
|
||||
"provider_schemas": {
|
||||
"registry.terraform.io/example/cloud": {
|
||||
"resource_schemas": {
|
||||
"cloud_service": {
|
||||
"block": {
|
||||
"attributes": {
|
||||
"name": {"type": "string", "required": True},
|
||||
"labels": {"type": ["map", "string"], "optional": True},
|
||||
"uri": {"type": "string", "computed": True},
|
||||
},
|
||||
"block_types": {
|
||||
"template": {
|
||||
"nesting_mode": "list",
|
||||
"max_items": 1,
|
||||
"block": {
|
||||
"attributes": {
|
||||
"cpu_boost": {
|
||||
"type": "bool",
|
||||
"optional": True,
|
||||
"description": "Boost CPU at start.",
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
"cloud_job": {"block": {"attributes": {"name": {"type": "string"}}}},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
PIN = {"cloud": {"source": "example/cloud", "version": "1.2.3"}}
|
||||
|
||||
|
||||
def schema_runner(monkeypatch) -> tuple[TerraformRunner, list[list[str]]]:
|
||||
commands: list[list[str]] = []
|
||||
|
||||
async def fake_run(self, args, cwd, env=None):
|
||||
commands.append(list(args))
|
||||
if args[1] == "init":
|
||||
assert (
|
||||
'cloud = { source = "example/cloud", version = "1.2.3" }'
|
||||
in (Path(cwd) / "versions.tf").read_text()
|
||||
)
|
||||
return 0, "ok"
|
||||
return 0, json.dumps(SCHEMA)
|
||||
|
||||
monkeypatch.setattr(TerraformRunner, "_run", fake_run)
|
||||
return TerraformRunner(providers=PIN), commands
|
||||
|
||||
|
||||
async def test_schema_lists_what_code_may_set_and_outlines_the_blocks(monkeypatch):
|
||||
runner, _ = schema_runner(monkeypatch)
|
||||
|
||||
result = await runner.schema("cloud_service")
|
||||
|
||||
assert result.message.startswith("cloud_service (example/cloud 1.2.3)")
|
||||
assert " name (string, required)" in result.message
|
||||
assert " labels (map of string, optional)" in result.message
|
||||
assert "uri" not in result.message # read-only: code cannot set it
|
||||
assert " template { } (list, at most 1): cpu_boost" in result.message
|
||||
|
||||
|
||||
async def test_schema_gives_one_block_in_full(monkeypatch):
|
||||
runner, _ = schema_runner(monkeypatch)
|
||||
|
||||
result = await runner.schema("cloud_service", block="template")
|
||||
missing = await runner.schema("cloud_service", block="template.containers")
|
||||
|
||||
assert " cpu_boost (bool, optional): Boost CPU at start." in result.message
|
||||
assert "'containers' is not a block there" in missing.message
|
||||
|
||||
|
||||
async def test_schema_says_where_a_setting_sits_or_that_it_cannot_be_set(monkeypatch):
|
||||
runner, _ = schema_runner(monkeypatch)
|
||||
|
||||
found = await runner.schema("cloud_service", find="cpu_boost")
|
||||
absent = await runner.schema("cloud_job", find="cpu_boost")
|
||||
|
||||
assert "template.cpu_boost (bool, optional)" in found.message
|
||||
assert "It cannot be set on this resource type." in absent.message
|
||||
|
||||
|
||||
async def test_schema_suggests_close_resource_types_and_is_loaded_once(monkeypatch):
|
||||
runner, commands = schema_runner(monkeypatch)
|
||||
|
||||
unknown = await runner.schema("cloud_servce")
|
||||
await runner.schema("cloud_job")
|
||||
|
||||
assert "Close matches: cloud_service" in unknown.message
|
||||
assert [command[1] for command in commands] == ["init", "providers"]
|
||||
assert "apply" not in [arg for command in commands for arg in command]
|
||||
|
||||
|
||||
async def test_schema_is_unavailable_without_a_pinned_provider():
|
||||
result = await TerraformRunner().schema("cloud_service")
|
||||
|
||||
assert "pins no Terraform provider" in result.message
|
||||
|
||||
|
||||
# --- registry ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -155,6 +260,7 @@ def test_local_tools_are_built_by_name_with_their_tier():
|
||||
"terraform_fmt",
|
||||
"terraform_validate",
|
||||
"terraform_plan",
|
||||
"terraform_schema",
|
||||
}
|
||||
with pytest.raises(KeyError, match="terraform_apply"):
|
||||
local_tools(["terraform_apply"], TerraformRunner())
|
||||
|
||||
Reference in New Issue
Block a user