fixed multile issues
Some checks failed
validation / verify (push) Failing after 8s

This commit is contained in:
2026-10-07 17:33:36 +01:00
parent 4c4d44ef87
commit 5bb22d984e
76 changed files with 5799 additions and 179 deletions

View File

@@ -271,6 +271,21 @@ async def test_model_timeout_is_set_on_every_request():
assert model.requests[0].config.http_options.timeout == 90_000
async def test_a_role_with_its_own_timeout_is_given_it_over_the_default():
model = ScriptedLlm(script=[[call("delegate")]])
runtime = AdkAgentRuntime(
skill_dirs=SKILLS, model_factory=lambda role: model, model_timeout_seconds=90
)
await runtime.run(
RoleSpec("orchestrator", "Coordinate.", budget=5, model_timeout_seconds=420),
TASK,
[Recorder().tool("delegate", TurnEnded("delegate"))],
)
assert model.requests[0].config.http_options.timeout == 420_000
class MeteredLlm(ScriptedLlm):
async def generate_content_async(
self, llm_request: LlmRequest, stream: bool = False

View File

@@ -19,7 +19,7 @@ def bundle():
def test_every_engine_role_and_the_pillar_advisor_is_defined(bundle):
roles = load_roles(AGENTS_DIR, bundle)
expected = {"orchestrator", PILLAR_ADVISOR}
expected = {"orchestrator", "reply-writer", PILLAR_ADVISOR}
expected |= {spec.specialist for spec in DEPENDENCY_GRAPH.values()}
assert set(roles) == expected
for role in roles.values():
@@ -46,12 +46,35 @@ def test_roles_take_skills_mcp_servers_and_models_from_the_bundle(bundle):
def test_roles_name_only_the_local_tools_they_need(bundle):
roles = load_roles(AGENTS_DIR, bundle)
assert roles["iac-author"].tools == ("terraform_fmt", "terraform_validate")
assert roles["target-architect"].tools == ()
assert roles["validator"].tools == ("terraform_validate", "terraform_plan")
assert roles["iac-author"].tools == (
"terraform_fmt",
"terraform_validate",
"terraform_schema",
)
# The architect only looks settings up; it writes and runs no code.
assert roles["target-architect"].tools == ("terraform_schema",)
assert roles["validator"].tools == (
"terraform_validate",
"terraform_plan",
"terraform_schema",
)
assert roles["requirements-analyst"].tools == ()
def test_roles_with_long_replies_are_given_longer_for_a_model_request(bundle):
roles = load_roles(AGENTS_DIR, bundle)
assert roles["target-architect"].model_timeout_seconds == 420
assert roles["decomposer"].model_timeout_seconds is None
def test_the_code_author_is_given_the_patterns_for_its_cloud_provider(bundle):
roles = load_roles(AGENTS_DIR, bundle)
assert "google-cloud-iac-patterns" in roles["iac-author"].skills
assert "google-cloud-iac-patterns" not in roles["validator"].skills
def test_preview_mcp_servers_can_be_excluded(bundle):
roles = load_roles(AGENTS_DIR, bundle, include_preview_mcp=False)

View File

@@ -475,3 +475,56 @@ async def test_a_scripted_user_drives_an_interactive_execution(tmp_path):
assert any("Approved." in reply for reply in replies)
assert any("europe-west2." in reply for reply in replies)
assert final.state["deliverables"]["requirements"]["content"]
# --- Jev as the judge -------------------------------------------------------------------
async def test_jev_judges_every_question_in_one_call_with_no_grading_in_the_state():
from app.adk.fake_runtime import FakeDecider
from app.adk.runtime import Answer
from eval.judge import DeciderJudge
decider = FakeDecider(
{
"covered": Answer(0.91, 0.82),
"missed": Answer(0.2, 0.6),
"specific": Answer(2.0, 0.7), # "mostly", on a scale of four
}
)
judge = DeciderJudge(decider)
scores = await judge.answer(
"## Reliability\nUse two zones.",
{
"covered": {"kind": "yes_no", "text": "Is redundancy addressed?"},
"missed": {"kind": "yes_no", "text": "Are backups tested?"},
"specific": {"kind": "graded", "text": "Is it specific?"},
},
)
assert scores == {"covered": 1.0, "missed": 0.0, "specific": 2 / 3}
[(state, questions, scope)] = decider.asked
assert state == "## Reliability\nUse two zones." # the document, nothing else
assert questions["covered"].kind == "noul"
assert questions["specific"].options == ("not at all", "weakly", "mostly", "fully")
assert scope == "evaluation"
async def test_a_judge_that_gives_no_answer_leaves_the_check_unanswered_not_passed():
from app.adk.runtime import NoDecider
from eval.judge import DeciderJudge
result = await score_execution(
CASE,
execution(),
HEADINGS,
passing_terraform,
judge=DeciderJudge(NoDecider()),
pillar_checklists=CHECKLISTS,
)
assert check(result, "pillar_checklists_covered").detail == (
"judge unavailable: Jev gave no answer"
)
assert not check(result, "recommendations_quality").passed

View File

@@ -15,7 +15,7 @@ from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
from app.tools.execution import specialist_tools
from app.tools.local import mermaid_check
from app.workflows.chain import validation_outcome
from app.workflows.chain import unimplementable_checks, validation_outcome
from app.workflows.deliverables import (
DEPENDENCY_GRAPH,
DeliverableKind,
@@ -327,6 +327,54 @@ def test_deployment_guidance_must_account_for_every_check():
)
def test_a_check_that_cannot_be_implemented_is_accounted_for_with_its_reason():
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
implementation = json.dumps(
[
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
{
"check": "K-2",
"unimplementable": "example_service_account has no such argument",
},
]
)
missing = check_submission(
K.DEPLOYMENT_GUIDANCE,
args["content"],
{**args["attachments"], "implementation.json": implementation},
upstream(*EVERYTHING_BEFORE_CODE),
)
assert missing == ()
assert unimplementable_checks(implementation) == [
{"check": "K-2", "reason": "example_service_account has no such argument"}
]
assert unimplementable_checks(SAMPLE_DATA["implementation.json"]) == []
def test_an_unimplementable_entry_needs_its_reason():
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
implementation = json.dumps(
[
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
{"check": "K-2", "unimplementable": ""},
]
)
missing = check_submission(
K.DEPLOYMENT_GUIDANCE,
args["content"],
{**args["attachments"], "implementation.json": implementation},
upstream(*EVERYTHING_BEFORE_CODE),
)
assert missing == (
"implementation.json[0]: needs unimplementable",
"implementation.json: Check K-2 is not accounted for",
)
def verdicts(**changes) -> str:
data = json.loads(SAMPLE_DATA["verdicts.json"])
data.update(changes)

View File

@@ -0,0 +1,598 @@
"""The Orchestrator as a router (ADR-0007): readings, a disposition, then a playbook.
No language model chooses an action here. Specialists are scripted; the Orchestrator
is the Router, with a scripted decider standing in for Jev, or none at all.
"""
import json
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import (
SAMPLE_DATA,
Call,
FakeAgentRuntime,
FakeDecider,
sample_submission,
)
from app.adk.runtime import Answer, NoDecider, RoleSpec
from app.agent import ExecutionEngine
from app.nodes.router import (
REPLY_WRITER,
Router,
RuntimeReplyWriter,
Thresholds,
)
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
K = DeliverableKind
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused by the router.", budget=40),
REPLY_WRITER: RoleSpec(REPLY_WRITER, "Write the reply.", budget=3),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def specialist(kind: DeliverableKind) -> str:
return DEPENDENCY_GRAPH[kind].specialist
def once() -> dict:
return {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
def validation(**changes) -> Call:
"""A validation submission whose verdicts differ from the all-pass sample."""
call = sample_submission(K.VALIDATION_RESULTS)
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
verdicts.update(changes)
return Call(
call.tool,
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
)
def failing(check: str = "K-1") -> Call:
checks = json.loads(SAMPLE_DATA["verdicts.json"])["checks"]
for item in checks:
if item["check"] == check:
item.update(verdict="fail", evidence="min instances is 1")
return validation(checks=checks)
def addition(reason: str) -> Call:
return validation(
proposed_additions=[
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": reason}
]
)
def engine_for(tmp_path, turns: dict, decider=None, writer: bool = False, **options):
runtime = FakeAgentRuntime(turns)
router = Router(
decider or NoDecider(),
RuntimeReplyWriter(runtime, ROLES[REPLY_WRITER]) if writer else None,
Thresholds(),
)
engine = ExecutionEngine(
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=router,
**options,
)
return engine, runtime
def actions(result) -> list[str]:
return [decision["action"] for decision in result.state["decisions"]]
def reopened(result) -> list[tuple[str, str]]:
return [
(event["subject"], event["detail"])
for event in result.state["ledger"]["events"]
if event["type"] == "reopened"
]
def approve(phase: str) -> dict:
return {"approval": {"phase": phase, "decision": "approve"}}
# --- the playbook on rules alone -------------------------------------------------------
async def test_an_execution_completes_with_no_model_choosing_an_action(tmp_path):
engine, runtime = engine_for(tmp_path, once())
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert "orchestrator" not in {run.role for run in runtime.runs}
assert actions(result) == [
"delegate", # requirements
"delegate", # technical decomposition
"request_approval",
"delegate", # product selection
"delegate", # design recommendations
"delegate", # Target architecture
"request_approval",
"delegate", # description and deployment guidance, together
"request_approval",
"delegate", # validation
"proceed",
"request_approval",
"delegate", # solution guide
"request_approval",
"finish",
]
groups = {decision["group"] for decision in result.state["decisions"]}
assert groups == {
"work is ready",
"a Phase is complete",
"validation decided",
"packaging is approved",
}
async def test_every_decision_is_recorded_with_its_turn_group_and_action(tmp_path):
engine, _ = engine_for(tmp_path, once())
result = await engine.start("Ingest webhooks", pre_approved=True)
first = result.state["decisions"][0]
assert first == {
"turn": 1,
"group": "work is ready",
"action": "delegate",
"judged": {},
}
assert [d["turn"] for d in result.state["decisions"]] == sorted(
d["turn"] for d in result.state["decisions"]
)
async def test_when_no_group_applies_a_pre_approved_execution_ends_with_warnings(
tmp_path,
):
turns = {specialist(K.REQUIREMENTS): [[]]} # stops without submitting
engine, _ = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
# The Specialist's question is relayed, answered automatically, and it stops
# again; the router never invents work.
assert result.status == "completed_with_warnings"
# --- Approvals: the summary is written from data, and only data approves ----------------
async def test_the_approval_summary_is_assembled_from_the_data(tmp_path):
engine, _ = engine_for(tmp_path, once())
first = await engine.start("Ingest webhooks")
design = await engine.resume(first.execution_id, approve("requirements_discovery"))
assert first.prompt["approval"]["summary"] == (
"The requirements_discovery Phase is ready: requirements, "
"technical_decomposition submitted. 2 requirements."
)
summary = design.prompt["approval"]["summary"]
assert "Target architecture: 2 Elements, 1 Relationships, 2 Checks." in summary
assert design.prompt["approval"]["target_architecture"]["checks"] == 2
async def test_a_model_writes_the_summary_but_decides_nothing(tmp_path):
turns = {
**once(),
REPLY_WRITER: [[Call("write_reply", {"text": "Two requirements, ready."})]],
}
engine, runtime = engine_for(tmp_path, turns, writer=True)
first = await engine.start("Ingest webhooks")
assert first.prompt["approval"]["summary"] == "Two requirements, ready."
writer_run = next(run for run in runtime.runs if run.role == REPLY_WRITER)
assert writer_run.tools == ("write_reply",)
assert writer_run.task.context["facts"]["deliverables"] == [
"requirements",
"technical_decomposition",
]
async def test_a_writer_that_fails_does_not_stop_the_approval(tmp_path):
engine, _ = engine_for(tmp_path, once(), writer=True) # no writer turn scripted
first = await engine.start("Ingest webhooks")
assert first.prompt["approval"]["summary"].startswith(
"The requirements_discovery Phase is ready"
)
async def test_a_text_reply_never_approves_and_the_approval_is_asked_again(tmp_path):
decider = FakeDecider(
{
"asks_change": Answer(0.03, 0.94),
"change_target": Answer("requirements", 0.5),
"change_clarity": Answer(0.0, 0.9),
}
)
engine, _ = engine_for(tmp_path, once(), decider)
first = await engine.start("Ingest webhooks")
again = await engine.resume(first.execution_id, {"reply": "Looks good to me."})
assert again.prompt["approval"]["phase"] == "requirements_discovery"
assert again.state["ledger"]["phases"]["requirements_discovery"] != "approved"
# --- validation: the code, the graph, or proceed ----------------------------------------
async def test_a_failed_check_sends_the_code_back_and_leaves_the_design_approved(
tmp_path,
):
turns = once()
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [
[failing("K-1")],
[sample_submission(K.VALIDATION_RESULTS)],
]
engine, runtime = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert reopened(result) == [
("deployment_guidance", "K-1 failed: min instances is 1")
]
assert "revise_code" in actions(result)
architect_runs = [
run for run in runtime.runs if run.role == specialist(K.TARGET_ARCHITECTURE)
]
assert len(architect_runs) == 1
async def test_a_check_the_code_cannot_meet_goes_back_to_the_graph_unvalidated(
tmp_path,
):
"""The code's author found the Check names a setting the resource lacks. No
validation is spent on it: the Target architecture is revised first."""
cannot = sample_submission(K.DEPLOYMENT_GUIDANCE)
implementation = json.loads(SAMPLE_DATA["implementation.json"])
implementation[1] = {"check": "K-2", "unimplementable": "no such argument"}
turns = once()
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
turns[specialist(K.DEPLOYMENT_GUIDANCE)] = [
[
Call(
cannot.tool,
{
**cannot.args,
"attachments": {
**cannot.args["attachments"],
"implementation.json": json.dumps(implementation),
},
},
)
],
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
]
engine, runtime = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert reopened(result) == [
(
"target_architecture",
(
"Check K-2 cannot be implemented as written (no such argument). "
"Rewrite it so it can be met, or remove it with a reason."
),
)
]
assert "the code cannot meet a Check" in {
decision["group"] for decision in result.state["decisions"]
}
validations = [
run for run in runtime.runs if run.role == specialist(K.VALIDATION_RESULTS)
]
assert len(validations) == 1
async def test_a_check_that_fails_twice_running_is_sent_back_to_the_graph(tmp_path):
"""The code was revised and the same Check failed again: the Check is suspect."""
turns = once()
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 3
turns[specialist(K.ARCHITECTURE_DESCRIPTION)] *= 2
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [
[failing("K-1")],
[failing("K-1")],
[sample_submission(K.VALIDATION_RESULTS)],
]
engine, _ = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
[first, second] = reopened(result)
assert first[0] == "deployment_guidance"
assert second[0] == "target_architecture"
assert "K-1 failed in two validations in a row" in second[1]
assert actions(result).count("revise_code") == 1
async def test_without_jev_a_gap_naming_a_real_requirement_revises_the_graph(tmp_path):
turns = once()
for kind in (
K.TARGET_ARCHITECTURE,
K.ARCHITECTURE_DESCRIPTION,
K.DEPLOYMENT_GUIDANCE,
):
turns[specialist(kind)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [
[addition("NFR-1 has no Check on timeouts")],
[sample_submission(K.VALIDATION_RESULTS)],
]
engine, _ = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
[(kind, reason)] = reopened(result)
assert kind == "target_architecture"
assert reason == (
"Add a Check on E-1: A timeout is set. (serves NFR-1 has no Check on timeouts)."
)
async def test_without_jev_a_gap_that_names_nothing_real_is_left(tmp_path):
turns = once()
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("it would be nice")]]
engine, _ = engine_for(tmp_path, turns)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert reopened(result) == []
async def test_jev_judges_whether_a_gap_is_worth_a_revision(tmp_path):
turns = once()
turns[specialist(K.VALIDATION_RESULTS)] = [[addition("NFR-1")]]
immaterial = FakeDecider({"gap_material": Answer(0.04, 0.92)})
engine, _ = engine_for(tmp_path, turns, immaterial)
result = await engine.start("Ingest webhooks", pre_approved=True)
# The rule alone would have revised the graph; the reading says it is not needed.
assert reopened(result) == []
judged = next(d for d in result.state["decisions"] if d["action"] == "proceed")
assert judged["judged"] == {"gap_material": {"value": 0.04, "confidence": 0.92}}
[(state, questions, scope)] = immaterial.asked
assert "A timeout is set." in state and scope == result.execution_id
assert questions["gap_material"].kind == "noul"
async def test_an_uncertain_gap_is_put_to_the_user_who_decides(tmp_path):
turns = once()
for kind in (
K.TARGET_ARCHITECTURE,
K.ARCHITECTURE_DESCRIPTION,
K.DEPLOYMENT_GUIDANCE,
):
turns[specialist(kind)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [
[addition("NFR-1")],
[sample_submission(K.VALIDATION_RESULTS)],
]
decider = FakeDecider(
{
"gap_material": Answer(0.5, 0.0), # the grey zone
"wants_revision": Answer(0.96, 0.92),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
result = await engine.start("Ingest webhooks")
for phase in (
"requirements_discovery",
"solution_architecture",
"solution_implementation",
):
result = await engine.resume(result.execution_id, approve(phase))
[question] = result.prompt["questions"]
assert (
"found gaps in the Target architecture: A timeout is set."
in (question["question"])
)
result = await engine.resume(result.execution_id, {"reply": "Yes, revise it."})
assert reopened(result)[0][0] == "target_architecture"
# Reopening the graph withdrew the design Approval, so it is asked for again.
assert result.prompt["approval"]["phase"] == "solution_architecture"
# --- the user's message: act, queue or close --------------------------------------------
def to_design_approval(engine):
async def run():
first = await engine.start("Ingest webhooks")
return await engine.resume(
first.execution_id, approve("requirements_discovery")
)
return run()
def changes(phase: str, feedback: str) -> dict:
return {"approval": {"phase": phase, "decision": "changes", "feedback": feedback}}
async def test_a_requested_change_is_routed_to_the_deliverable_it_affects(tmp_path):
turns = once()
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
decider = FakeDecider(
{
"change_target": Answer("design_recommendations", 0.88),
"change_clarity": Answer(1.8, 0.8),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
design = await to_design_approval(engine)
result = await engine.resume(
design.execution_id,
changes("solution_architecture", "Use customer-managed keys on the topic."),
)
assert reopened(result) == [
("design_recommendations", "Use customer-managed keys on the topic.")
]
# Product selection was not touched; the design is asked for again once redone.
assert result.state["ledger"]["deliverables"]["product_selection"]["revisions"] == 0
assert result.prompt["approval"]["phase"] == "solution_architecture"
[(state, questions, _)] = decider.asked
assert "The user's message: Use customer-managed keys on the topic." in state
assert questions["change_target"].options == (
"requirements",
"technical_decomposition",
"product_selection",
"design_recommendations",
"target_architecture",
)
assert "asks_change" not in questions # a structured request is known to be one
async def test_an_unclear_change_is_confirmed_with_the_user_before_anything_moves(
tmp_path,
):
turns = once()
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
decider = FakeDecider(
{
"change_target": Answer("product_selection", 0.31), # not relied on
"change_clarity": Answer(1.5, 0.7),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
design = await to_design_approval(engine)
asked = await engine.resume(
design.execution_id, changes("solution_architecture", "Tighten it up.")
)
assert reopened(asked) == []
[question] = asked.prompt["questions"]
assert question["question"].startswith("Do you want product_selection changed?")
result = await engine.resume(
asked.execution_id,
{"reply": "No, change the target architecture: add a Check on timeouts."},
)
[(kind, reason)] = reopened(result)
assert kind == "target_architecture"
assert "Tighten it up." in reason and "add a Check on timeouts" in reason
async def test_a_free_text_reply_that_asks_for_a_change_is_acted_on(tmp_path):
turns = once()
turns[specialist(K.REQUIREMENTS)] = [
[sample_submission(K.REQUIREMENTS)],
[sample_submission(K.REQUIREMENTS)],
]
turns[specialist(K.TECHNICAL_DECOMPOSITION)] *= 2
decider = FakeDecider(
{
"asks_change": Answer(0.93, 0.86),
"change_target": Answer("requirements", 0.9),
"change_clarity": Answer(1.9, 0.8),
}
)
engine, _ = engine_for(tmp_path, turns, decider)
first = await engine.start("Ingest webhooks")
result = await engine.resume(
first.execution_id, {"reply": "Retention must be 7 years, not 30 days."}
)
assert reopened(result) == [
("requirements", "Retention must be 7 years, not 30 days.")
]
async def test_without_jev_a_change_is_acted_on_only_when_the_deliverable_is_named(
tmp_path,
):
turns = once()
turns[specialist(K.DESIGN_RECOMMENDATIONS)] *= 2
turns[specialist(K.TARGET_ARCHITECTURE)] *= 2
engine, _ = engine_for(tmp_path, turns)
design = await to_design_approval(engine)
unnamed = await engine.resume(
design.execution_id, changes("solution_architecture", "Make it cheaper.")
)
assert reopened(unnamed) == []
assert unnamed.prompt["questions"][0]["question"].startswith(
"Which Deliverable should change"
)
named = await engine.resume(
unnamed.execution_id, {"reply": "The design recommendations."}
)
[(kind, reason)] = reopened(named)
assert kind == "design_recommendations"
assert reason == (
"Make it cheaper.\nThe user then said: The design recommendations."
)
# --- permission is the Ledger's, whatever was decided ------------------------------------
async def test_past_the_revision_limit_the_user_decides(tmp_path):
turns = once()
turns[specialist(K.DEPLOYMENT_GUIDANCE)] *= 2
turns[specialist(K.VALIDATION_RESULTS)] = [[failing("K-2")], [failing("K-1")]]
engine, _ = engine_for(tmp_path, turns, max_revisions=1)
result = await engine.start("Ingest webhooks")
for phase in (
"requirements_discovery",
"solution_architecture",
"solution_implementation",
"solution_implementation",
):
result = await engine.resume(result.execution_id, approve(phase))
assert len(reopened(result)) == 1
[question] = result.prompt["questions"]
assert (
"deployment_guidance has reached its revision limit (1)"
in (question["question"])
)

View File

@@ -0,0 +1,480 @@
"""A Specialist's working copy, and what each Specialist is given to read.
Three things made an Execution slow: a Specialist rewrote a whole Deliverable in one
reply, sent every file again on each tool call, and read every earlier Deliverable in
full. A working copy and trimmed reading remove each of them.
"""
import json
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import (
SAMPLE_DATA,
Call,
FakeAgentRuntime,
sample_submission,
)
from app.adk.runtime import NoDecider, RoleSpec, Submitted, ToolResult
from app.agent import ExecutionEngine
from app.nodes.router import Router, Thresholds
from app.recorder import InMemoryRecorder
from app.tools.execution import specialist_tools
from app.tools.local import TerraformRunner, local_tools
from app.tools.toolkit import make_tool_factory
from app.tools.workspace import file_tools, graph_tools
from app.workflows.deliverables import (
DEPENDENCY_GRAPH,
DeliverableKind,
dependencies_for,
)
from app.workflows.target_architecture import parse_graph, render_tables
K = DeliverableKind
GRAPH = json.loads(SAMPLE_DATA["target-architecture.json"])
UPSTREAM = {
"requirements": {
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
},
"technical_decomposition": {
"attachments": {"components.json": SAMPLE_DATA["components.json"]}
},
"product_selection": {
"attachments": {"products.json": SAMPLE_DATA["products.json"]}
},
"design_recommendations": {
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
},
"target_architecture": {
"attachments": {
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
}
},
}
def tools_of(kind, working, upstream=UPSTREAM, extra=()):
return {
tool.name: tool
for tool in specialist_tools(
kind, extra, allow_cloud_read=False, upstream=upstream, working=working
)
}
# --- the Target architecture is written once -------------------------------------------
def test_tables_are_rendered_from_the_graph():
tables = render_tables(parse_graph(SAMPLE_DATA["target-architecture.json"]))
assert "## Elements" in tables and "## Relationships" in tables
assert (
"| E-1 | Ingress service | Example Run | example_run_service | C-1 |" in tables
)
assert "| R-1 | Ingress service | runs as | Ingress identity |" in tables
assert "| K-1 | E-1 | The service runs at least two instances. | NFR-1 |" in tables
def test_a_table_cell_cannot_break_its_row():
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
graph["checks"][0]["statement"] = "a | b\nc"
assert "| a \\| b c |" in render_tables(parse_graph(json.dumps(graph)))
async def test_the_engine_writes_the_tables_and_the_diagram_on_submission():
tools = tools_of(K.TARGET_ARCHITECTURE, {})
result = await tools["submit_deliverable"].handler(
content="## Design rationale\nOne service, one identity.\n",
attachments={
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
},
)
assert isinstance(result.outcome, Submitted)
assert result.outcome.content.startswith("## Design rationale")
assert "## Checks" in result.outcome.content
assert result.outcome.attachments["architecture.mmd"].startswith("flowchart")
async def test_the_graph_is_built_a_few_items_at_a_time_and_attached_on_submission():
working: dict = {}
tools = tools_of(K.TARGET_ARCHITECTURE, working)
await tools["graph_put"].handler(elements=GRAPH["elements"])
await tools["graph_put"].handler(relationships=GRAPH["relationships"])
put = await tools["graph_put"].handler(checks=GRAPH["checks"])
assert put.message == (
"Added 2, replaced 0. The graph has 2 Elements, 1 Relationships and 2 Checks."
)
result = await tools["submit_deliverable"].handler(
content="## Design rationale\nReasons.\n"
)
assert isinstance(result.outcome, Submitted)
submitted = json.loads(result.outcome.attachments["target-architecture.json"])
assert [item["id"] for item in submitted["checks"]] == ["K-1", "K-2"]
async def test_a_revision_amends_the_graph_that_was_submitted():
working: dict = {}
tools = tools_of(K.TARGET_ARCHITECTURE, working)
await tools["submit_deliverable"].handler(
content="## Design rationale\nReasons.\n",
attachments={
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
},
)
# A later attempt, with the working memory the Execution kept.
tools = tools_of(K.TARGET_ARCHITECTURE, working)
changed = {**GRAPH["checks"][0], "statement": "min_instance_count is at least 2"}
put = await tools["graph_put"].handler(checks=[changed])
read = await tools["graph_read"].handler()
assert put.message.startswith("Added 0, replaced 1.")
assert "min_instance_count is at least 2" in read.message
assert len(json.loads(read.message)["checks"]) == 2
async def test_graph_items_can_be_removed_and_an_item_without_an_id_is_refused():
working: dict = {}
tools = {tool.name: tool for tool in graph_tools(working)}
await tools["graph_put"].handler(
elements=GRAPH["elements"],
not_applicable=[{"recommendation": "DR-9", "reason": "Single region."}],
)
removed = await tools["graph_remove"].handler(ids=["E-2", "DR-9"])
refused = await tools["graph_put"].handler(checks=[{"statement": "no id"}])
assert removed.message.startswith("Removed 2.")
assert refused.message.startswith("Refused") and "checks[0]" in refused.message
assert [e["id"] for e in working["graph"]["elements"]] == ["E-1"]
# --- files are written once and changed in place ----------------------------------------
async def test_a_file_is_written_once_and_changed_in_place():
working: dict = {}
tools = {tool.name: tool for tool in file_tools(working)}
await tools["write_file"].handler(
path="terraform/main.tf", content="a = 1\nb = 1\n"
)
edited = await tools["edit_file"].handler(
path="terraform/main.tf", old="b = 1", new="b = 2"
)
listing = await tools["read_file"].handler()
assert edited.message == "Edited terraform/main.tf."
assert working["files"] == {"terraform/main.tf": "a = 1\nb = 2\n"}
assert "- terraform/main.tf (2 lines)" in listing.message
async def test_an_edit_that_does_not_match_exactly_once_changes_nothing():
working: dict = {"files": {"main.tf": "x = 1\nx = 1\n"}}
tools = {tool.name: tool for tool in file_tools(working)}
twice = await tools["edit_file"].handler(path="main.tf", old="x = 1", new="x = 2")
never = await tools["edit_file"].handler(path="main.tf", old="y", new="z")
absent = await tools["edit_file"].handler(path="other.tf", old="x", new="y")
unsafe = await tools["write_file"].handler(path="../main.tf", content="x")
assert "it matches 2" in twice.message and "it matches 0" in never.message
assert absent.message.startswith("Refused") and unsafe.message.startswith("Refused")
assert working["files"] == {"main.tf": "x = 1\nx = 1\n"}
async def test_the_working_copy_is_what_is_submitted():
working: dict = {}
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
for path in ("terraform/main.tf", "terraform/variables.tf"):
await tools["write_file"].handler(path=path, content=f"# {path}\n")
await tools["write_file"].handler(
path="implementation.json", content=SAMPLE_DATA["implementation.json"]
)
content = sample_submission(K.DEPLOYMENT_GUIDANCE).args["content"]
result = await tools["submit_deliverable"].handler(content=content)
assert isinstance(result.outcome, Submitted)
assert set(result.outcome.attachments) == {
"terraform/main.tf",
"terraform/variables.tf",
"implementation.json",
}
async def test_a_file_passed_on_submission_replaces_its_working_copy():
working: dict = {"files": {"terraform/main.tf": "old\n"}}
tools = tools_of(K.DEPLOYMENT_GUIDANCE, working)
call = sample_submission(K.DEPLOYMENT_GUIDANCE)
result = await tools["submit_deliverable"].handler(**call.args)
assert isinstance(result.outcome, Submitted)
assert result.outcome.attachments["terraform/main.tf"] != "old\n"
assert working["files"]["terraform/main.tf"] != "old\n"
def test_only_a_specialist_that_owns_files_can_write_them():
assert "write_file" in tools_of(K.DEPLOYMENT_GUIDANCE, {})
assert "write_file" not in tools_of(K.VALIDATION_RESULTS, {})
assert "graph_put" in tools_of(K.TARGET_ARCHITECTURE, {})
assert "graph_put" not in tools_of(K.DEPLOYMENT_GUIDANCE, {})
# --- Terraform runs on the working copy -------------------------------------------------
class Runner(TerraformRunner):
def __init__(self) -> None:
super().__init__()
self.validated: list[dict[str, str]] = []
async def validate(self, files):
self.validated.append(dict(files))
return ToolResult("terraform validate passed:\n")
async def formatted(self, files):
return {name: content.strip() + "\n" for name, content in files.items()}
async def test_terraform_checks_take_no_files_when_there_is_a_working_copy():
runner = Runner()
files = {"terraform/main.tf": "x = 1\n", "implementation.json": "[]"}
(validate,) = local_tools(["terraform_validate"], runner, files)
await validate.handler()
assert validate.parameters["required"] == []
assert "pass no files" in validate.description
# Only the Terraform files are checked.
assert runner.validated == [{"terraform/main.tf": "x = 1\n"}]
async def test_formatting_changes_the_working_copy_and_returns_only_names():
files = {"terraform/main.tf": " x = 1 \n"}
(fmt,) = local_tools(["terraform_fmt"], Runner(), files)
result = await fmt.handler()
assert files == {"terraform/main.tf": "x = 1\n"}
assert result.message == (
"terraform fmt reformatted 1 file(s) in your working copy: terraform/main.tf."
)
async def test_files_passed_anyway_are_written_to_the_working_copy_first():
runner = Runner()
files: dict[str, str] = {}
(validate,) = local_tools(["terraform_validate"], runner, files)
await validate.handler(files=[{"path": "terraform/main.tf", "content": "y"}])
assert files == {"terraform/main.tf": "y"}
assert runner.validated == [files]
def roles_with(tools_by_role: dict[str, tuple[str, ...]]) -> dict[str, RoleSpec]:
return {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(
spec.specialist,
"Produce.",
budget=15,
tools=tools_by_role.get(spec.specialist, ()),
)
for spec in DEPENDENCY_GRAPH.values()
},
}
async def test_the_validator_checks_the_submitted_code_without_being_sent_it(tmp_path):
runner = Runner()
roles = roles_with({"validator": ("terraform_validate",)})
turns = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
turns["validator"] = [
[Call("terraform_validate", {}), sample_submission(K.VALIDATION_RESULTS)]
]
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
tool_factory=make_tool_factory(
runtime,
roles,
{},
local_tools=lambda working_copy: local_tools(
["terraform_validate"], runner, working_copy
),
),
)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
assert runner.validated == [
{
"terraform/main.tf": "# terraform/main.tf\n",
"terraform/variables.tf": "# terraform/variables.tf\n",
}
]
# The code is the code author's: validation attaches its verdicts and nothing else.
validation = result.state["deliverables"]["validation_results"]
assert set(validation["attachments"]) == {"verdicts.json"}
# --- each Specialist reads what it works from -------------------------------------------
def submitted() -> dict[str, dict]:
return {
kind.value: {
"content": f"# {kind}\nProse.\n",
"attachments": {
name: SAMPLE_DATA.get(name, f"# {name}\n") for name in spec.attachments
},
}
for kind, spec in DEPENDENCY_GRAPH.items()
}
def test_the_code_is_written_from_the_graph_and_the_ids_alone():
given = dependencies_for(K.DEPLOYMENT_GUIDANCE, submitted())
assert given["target_architecture"] == {
"attachments": {
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
}
}
assert all("content" not in deliverable for deliverable in given.values())
assert set(given["requirements"]["attachments"]) == {"requirements.json"}
def test_a_direct_dependency_is_read_whole_and_the_rest_as_data():
given = dependencies_for(K.TARGET_ARCHITECTURE, submitted())
assert "content" in given["product_selection"]
assert "content" in given["design_recommendations"]
assert "content" not in given["requirements"]
assert "content" not in given["technical_decomposition"]
def test_the_validator_reads_the_code_whole_and_the_graph_as_data():
given = dependencies_for(K.VALIDATION_RESULTS, submitted())
assert "terraform/main.tf" in given["deployment_guidance"]["attachments"]
assert "content" not in given["target_architecture"]
# A Deliverable with no data file has nothing to trim to, so it comes whole.
assert "content" in given["architecture_description"]
def test_the_solution_guide_is_assembled_from_every_deliverable_whole():
given = dependencies_for(K.SOLUTION_GUIDE, submitted())
assert len(given) == len(DEPENDENCY_GRAPH) - 1
assert all("content" in deliverable for deliverable in given.values())
async def test_a_specialist_is_told_what_its_working_copy_already_holds(tmp_path):
turns = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
failing = sample_submission(K.VALIDATION_RESULTS)
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
verdicts["checks"][0].update(verdict="fail", evidence="min instances is 1")
turns["validator"] = [
[
Call(
failing.tool,
{
**failing.args,
"attachments": {"verdicts.json": json.dumps(verdicts)},
},
)
],
[sample_submission(K.VALIDATION_RESULTS)],
]
turns["iac-author"] = [[sample_submission(K.DEPLOYMENT_GUIDANCE)]] * 2
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
runtime=runtime,
roles=roles_with({}),
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
await engine.start("Ingest webhooks", pre_approved=True)
first, second = (run for run in runtime.runs if run.role == "iac-author")
assert first.task.context["working_copy"] is None
assert second.task.context["working_copy"] == {
"files": ["implementation.json", "terraform/main.tf", "terraform/variables.tf"]
}
async def test_a_deliverable_given_as_data_can_be_read_whole_on_demand():
library = {
"requirements": {
"content": "# requirements\nProse.\n",
"attachments": {"requirements.json": "[]"},
}
}
tools = {
tool.name: tool
for tool in specialist_tools(
K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False, library=library
)
}
read = await tools["read_deliverable"].handler(kind="requirements")
refused = await tools["read_deliverable"].handler(kind="solution_guide")
assert (
read.message.startswith("# requirements")
and "requirements.json" in read.message
)
assert refused.message.startswith("Refused")
# With nothing trimmed there is nothing to read back.
assert "read_deliverable" not in tools_of(K.REQUIREMENTS, {})
async def test_the_engine_offers_the_trimmed_deliverables_for_reading(tmp_path):
turns = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
runtime=runtime,
roles=roles_with({}),
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
await engine.start("Ingest webhooks", pre_approved=True)
author = next(run for run in runtime.runs if run.role == "iac-author")
packager = next(run for run in runtime.runs if run.role == "packager")
assert "read_deliverable" in author.tools
assert "content" not in author.task.context["dependencies"]["requirements"]
assert "read_deliverable" not in packager.tools

View File

@@ -12,6 +12,7 @@ def settings(tmp_path, **overrides) -> Settings:
values: dict[str, Any] = {
"CHECKPOINTER": "memory",
"MCP_ENABLED": False,
"ORCHESTRATOR": "model",
"ORCHESTRATOR_BUDGET": 9,
"SPECIALIST_BUDGET": 4,
}
@@ -50,3 +51,42 @@ async def test_preview_mcp_servers_follow_the_setting(tmp_path):
assert "asset_inventory" not in servers
assert "compute" in servers
async def test_the_assembly_routes_by_default_and_runs_without_a_jev_key(tmp_path):
"""With no Jev key the router follows its playbook on rules alone."""
from app.adk.fake_runtime import sample_submission
from app.adk.runtime import NoDecider
from app.workflows.deliverables import DeliverableKind
runtime = FakeAgentRuntime(
{"requirements-analyst": [[sample_submission(DeliverableKind.REQUIREMENTS)]]}
)
async with open_assembly(
settings(tmp_path, ORCHESTRATOR="router", TYPESAFE_API_KEY=None),
runtime=runtime,
recorder=InMemoryRecorder(),
output_root=tmp_path,
) as assembly:
interaction = await assembly.engine.start("Design ingestion")
assert isinstance(assembly.decider, NoDecider)
# The router delegated requirements with no Orchestrator model, then the
# decomposer (unscripted here) stopped and its question reached the user.
assert [run.role for run in runtime.runs] == ["requirements-analyst"]
assert interaction.status == "awaiting_user"
assert "reply-writer" in assembly.roles
async def test_an_unknown_orchestrator_setting_is_refused(tmp_path):
import pytest
with pytest.raises(ValueError, match="ORCHESTRATOR must be router or model"):
async with open_assembly(
settings(tmp_path, ORCHESTRATOR="llm"),
runtime=FakeAgentRuntime({}),
recorder=InMemoryRecorder(),
output_root=tmp_path,
):
pass

View File

@@ -0,0 +1,62 @@
"""The IaC patterns Skill: every pattern validates against the pinned provider.
A pattern that does not validate teaches the mistake it was written to prevent, so
each reference's Terraform is run through `terraform validate` with the Skill's own
versions file. The provider comes from the plugin cache; with no cache and no
network the test is skipped, not failed.
"""
import re
import shutil
from pathlib import Path
import pytest
from app.config import Settings
from app.providers import load_provider
from app.tools.local import TerraformRunner
SKILL = "google-cloud-iac-patterns"
ROOT = Path(__file__).resolve().parent.parent / "skills" / SKILL
REFERENCES = sorted((ROOT / "references").glob("*.md"))
_HCL = re.compile(r"^```hcl\n(.*?)^```", re.MULTILINE | re.DOTALL)
needs_terraform = pytest.mark.skipif(
shutil.which("terraform") is None, reason="terraform CLI not installed"
)
def _runner() -> TerraformRunner:
return TerraformRunner(plugin_cache_dir=Settings().terraform_plugin_cache())
def test_every_reference_is_listed_in_the_skill_and_holds_a_pattern():
skill = (ROOT / "SKILL.md").read_text(encoding="utf-8")
assert REFERENCES
for reference in REFERENCES:
assert f"references/{reference.name}" in skill
assert _HCL.findall(reference.read_text(encoding="utf-8")), reference.name
listed = set(re.findall(r"references/([\w-]+\.md)", skill))
assert listed == {reference.name for reference in REFERENCES}
def test_the_skill_pins_the_version_the_provider_bundle_pins():
pin = load_provider("gcp").terraform_providers["google"]
versions = (ROOT / "assets" / "versions.tf").read_text(encoding="utf-8")
skill = (ROOT / "SKILL.md").read_text(encoding="utf-8")
assert f'source = "{pin["source"]}"' in versions
assert f'version = "{pin["version"]}"' in versions
assert f"{pin['source']} {pin['version']}" in skill
@needs_terraform
@pytest.mark.parametrize("reference", REFERENCES, ids=lambda path: path.stem)
async def test_pattern_validates(reference: Path):
files = {
"terraform/versions.tf": (ROOT / "assets" / "versions.tf").read_text(),
"terraform/main.tf": "\n".join(_HCL.findall(reference.read_text())),
}
result = await _runner().validate(files)
if result.message.startswith("terraform init failed"):
pytest.skip("the pinned provider could not be installed")
assert result.message.startswith("terraform validate passed"), result.message

View File

@@ -218,3 +218,55 @@ def test_bundle_places_each_local_tool_in_its_own_tier():
for tier, names in bundle.tools.items():
for name in names:
assert LOCAL_TOOLS[name][0] == tier, name
def test_bundle_pins_terraform_providers_and_role_timeouts(tmp_path):
config = minimal_bundle(
tmp_path,
models={"default": "gemini/m", "timeout_seconds": {"target-architect": 420}},
terraform={
"required_providers": {
"cloud": {"source": "example/cloud", "version": "1.2.3"}
}
},
)
write_bundle(tmp_path, config)
bundle = load_provider("test", root=tmp_path)
assert bundle.terraform_providers == {
"cloud": {"source": "example/cloud", "version": "1.2.3"}
}
assert bundle.models.timeout_for("target-architect") == 420
assert bundle.models.timeout_for("decomposer") is None
@pytest.mark.parametrize(
"overrides, message",
[
(
{"terraform": {"required_providers": {"cloud": {"source": "x/cloud"}}}},
"needs a 'source' and an exact 'version'",
),
(
{"models": {"default": "gemini/m", "timeout_seconds": {"x": 0}}},
"must be positive",
),
(
{"models": {"default": "gemini/m", "timeout_seconds": {"x": "slow"}}},
"must be a number",
),
],
)
def test_bad_terraform_pins_and_timeouts_are_rejected(tmp_path, overrides, message):
write_bundle(tmp_path, minimal_bundle(tmp_path, **overrides))
with pytest.raises(ProviderConfigError, match=message):
load_provider("test", root=tmp_path)
def test_gcp_bundle_pins_one_exact_google_provider_version():
pin = load_provider("gcp").terraform_providers["google"]
assert pin["source"] == "hashicorp/google"
assert pin["version"].replace(".", "").isdigit() # exact, not a range

417
tests/tools/test_jev.py Normal file
View File

@@ -0,0 +1,417 @@
"""The Jev decision client: typed answers within strict limits, and a fallback always.
Every test runs the real TypeSafe SDK against a stand-in transport; nothing here calls
the live service.
"""
import asyncio
import json
import logging
import httpx2
from app.adk.runtime import Answer, NoDecider, Question
from app.agent import open_decider
from app.config import Settings
from app.tools.jev import JevDecider, JevLimits, estimate_tokens
URGENT = Question("noul", "Does this need attention right now?")
ROUTE = Question(
"choice", "What comes next?", ("revise_code", "revise_graph", "proceed")
)
SEVERITY = Question("score", "How much does it matter?", ("low", "medium", "high"))
ALL = {"urgent": URGENT, "route": ROUTE, "severity": SEVERITY}
ANSWERS = {
"urgent": {"type": "noul", "noul": 0.97},
"route": {
"type": "choice",
"choice": "revise_code",
"confidence": 0.81,
"probabilities": {"revise_code": 0.81, "revise_graph": 0.12, "proceed": 0.07},
},
"severity": {
"type": "score",
"score": 1.6,
"confidence": 0.7,
"legend": {"0": "low", "1": "medium", "2": "high"},
"probabilities": {"0": 0.1, "1": 0.2, "2": 0.7},
},
}
class Clock:
"""Time that moves only when something sleeps."""
def __init__(self) -> None:
self.now = 1000.0
self.slept: list[float] = []
def __call__(self) -> float:
return self.now
async def sleep(self, seconds: float) -> None:
self.slept.append(seconds)
self.now += seconds
def ok(*names: str, tokens: int = 40) -> httpx2.Response:
answers = {name: ANSWERS[name] for name in names} if names else ANSWERS
return httpx2.Response(
200,
json={
"model": "jev-latest",
"answers": answers,
"usage": {"input_tokens": tokens, "output_tokens": 0},
},
)
class Service:
"""A stand-in Jev: replies from a queue, then with `default`; records requests.
A reply answers only the questions the request asked, as the service does.
"""
def __init__(self, *replies, default=None) -> None:
self.replies = list(replies)
self.default = default
self.requests: list[httpx2.Request] = []
def __call__(self, request: httpx2.Request) -> httpx2.Response:
self.requests.append(request)
reply = self.replies.pop(0) if self.replies else self.default
if reply is None:
return ok(*json.loads(request.content)["questions"])
if isinstance(reply, Exception):
raise reply
return reply
@property
def bodies(self) -> list[dict]:
return [json.loads(request.content) for request in self.requests]
def decider(service: Service, clock: Clock | None = None, **limits) -> JevDecider:
clock = clock or Clock()
limits.setdefault("backoff_base", 0.0) # no real waiting between SDK retries
return JevDecider(
api_key="tsk_secret",
base_url="https://jev.example",
limits=JevLimits(**limits),
transport=httpx2.MockTransport(service),
clock=clock,
sleep=clock.sleep,
)
# --- the request and the typed answers -------------------------------------------------
async def test_one_request_carries_the_state_and_every_question():
service = Service()
jev = decider(service)
answers = await jev.decide("K-30 failed twice.", ALL, scope="exec-1")
[request] = service.requests
assert request.method == "POST"
assert str(request.url) == "https://jev.example/v1/systemone"
assert request.headers["Authorization"] == "Bearer tsk_secret"
body = service.bodies[0]
assert body["state"] == "K-30 failed twice."
assert body["questions"] == {
"urgent": {"type": "noul", "instructions": URGENT.instructions},
"route": {
"type": "choice",
"instructions": ROUTE.instructions,
"criteria": {"revise_code": None, "revise_graph": None, "proceed": None},
},
"severity": {
"type": "score",
"instructions": SEVERITY.instructions,
"criteria": ["low", "medium", "high"],
},
}
assert answers["route"] == Answer(
"revise_code",
0.81,
{"revise_code": 0.81, "revise_graph": 0.12, "proceed": 0.07},
)
assert answers["severity"] == Answer(1.6, 0.7, {"0": 0.1, "1": 0.2, "2": 0.7})
# A yes-or-no answer is a probability; its confidence is its distance from even.
assert answers["urgent"].value == 0.97
assert round(answers["urgent"].confidence, 2) == 0.94
assert answers["urgent"].is_yes() and not Answer(0.2, 0.6).is_yes()
async def test_a_choice_outside_the_options_or_a_missing_answer_is_no_decision():
invented = httpx2.Response(
200,
json={
"model": "jev-latest",
"usage": {"input_tokens": 1, "output_tokens": 0},
"answers": {
"route": {
"type": "choice",
"choice": "delete_everything",
"confidence": 0.99,
"probabilities": {},
}
},
},
)
assert await decider(Service(invented)).decide("s", {"route": ROUTE}) is None
assert await decider(Service(ok("urgent"))).decide("s", ALL) is None
assert (
await decider(Service(httpx2.Response(200, text="not json"))).decide(
"s", {"urgent": URGENT}
)
is None
)
async def test_no_questions_needs_no_request():
service = Service()
assert await decider(service).decide("state", {}) == {}
assert service.requests == []
# --- size --------------------------------------------------------------------------------
async def test_a_request_over_the_size_limit_is_refused_locally_never_truncated():
service = Service()
jev = decider(service, max_tokens=100, max_question_tokens=100)
state = "x" * 600
assert estimate_tokens(state) > 100
assert await jev.decide(state, {"urgent": URGENT}) is None
assert service.requests == []
assert jev.records[-1].outcome == "too_large"
# --- timeouts and retries (the SDK's, configured here) ------------------------------------
async def test_temporary_failures_are_retried_then_answered():
service = Service(httpx2.Response(503), httpx2.ConnectTimeout("slow"))
jev = decider(service)
assert await jev.decide("s", {"urgent": URGENT}) is not None
assert len(service.requests) == 3
async def test_a_rate_limit_that_outlasts_the_retries_lowers_the_clients_own_rate():
service = Service(default=httpx2.Response(429))
jev = decider(service, retries=1)
assert await jev.decide("s", {"urgent": URGENT}) is None
assert len(service.requests) == 2
assert jev.records[-1].rate_limited
assert jev._rate == JevLimits().requests_per_second / 2
async def test_a_rejected_request_is_not_retried():
service = Service(default=httpx2.Response(400, json={"error": "bad question"}))
jev = decider(service)
assert await jev.decide("s", {"urgent": URGENT}) is None
assert len(service.requests) == 1
async def test_retries_stop_at_the_budget_and_the_caller_falls_back():
service = Service(default=httpx2.Response(503))
jev = decider(service, retries=2)
assert await jev.decide("s", {"urgent": URGENT}) is None
assert len(service.requests) == 3
assert jev.records[-1].outcome == "failed"
# --- rate and concurrency ----------------------------------------------------------------
async def test_calls_beyond_the_burst_wait_for_the_bucket_to_refill():
clock = Clock()
service = Service()
jev = decider(service, clock, burst=2, requests_per_second=4)
for index in range(3):
await jev.decide(f"state {index}", {"urgent": URGENT})
assert len(service.requests) == 3
assert clock.slept == [0.25] # one token at four a second
async def test_calls_in_flight_never_exceed_the_cap():
in_flight = peak = 0
async def slow(request: httpx2.Request) -> httpx2.Response:
nonlocal in_flight, peak
in_flight += 1
peak = max(peak, in_flight)
await asyncio.sleep(0.01)
in_flight -= 1
return ok("urgent")
jev = JevDecider(
api_key="k",
base_url="https://jev.example",
limits=JevLimits(max_in_flight=2, burst=50),
transport=httpx2.MockTransport(slow),
)
await asyncio.gather(
*(jev.decide(f"state {index}", {"urgent": URGENT}) for index in range(8))
)
assert peak == 2
# --- circuit breaker ---------------------------------------------------------------------
async def test_repeated_failures_open_the_circuit_and_one_probe_closes_it():
clock = Clock()
service = Service(default=httpx2.Response(503))
jev = decider(
service, clock, retries=0, breaker_failures=3, breaker_cooldown=60, burst=50
)
for index in range(3):
assert await jev.decide(f"s{index}", {"urgent": URGENT}) is None
sent = len(service.requests)
# Open: the service is not called at all.
assert await jev.decide("s-open", {"urgent": URGENT}) is None
assert len(service.requests) == sent
assert jev.records[-1].outcome == "breaker_open"
# After the cool-down a single call probes; its success closes the circuit.
clock.now += 61
service.default = None
assert await jev.decide("s-probe", {"urgent": URGENT}) is not None
assert await jev.decide("s-after", {"urgent": URGENT}) is not None
async def test_a_failed_probe_reopens_the_circuit():
clock = Clock()
service = Service(default=httpx2.Response(503))
jev = decider(
service, clock, retries=0, breaker_failures=2, breaker_cooldown=60, burst=50
)
for index in range(2):
await jev.decide(f"s{index}", {"urgent": URGENT})
clock.now += 61
await jev.decide("probe", {"urgent": URGENT})
sent = len(service.requests)
assert await jev.decide("again", {"urgent": URGENT}) is None
assert len(service.requests) == sent
# --- cache and budget --------------------------------------------------------------------
async def test_the_same_question_about_the_same_state_is_answered_from_memory():
service = Service()
jev = decider(service)
first = await jev.decide("unchanged", {"route": ROUTE})
second = await jev.decide("unchanged", {"route": ROUTE})
await jev.decide("changed", {"route": ROUTE})
assert first == second
assert len(service.requests) == 2
assert [r.outcome for r in jev.records] == ["answered", "cached", "answered"]
async def test_each_execution_has_a_budget_of_calls_and_then_falls_back(caplog):
service = Service()
jev = decider(service, budget_calls=2, burst=50)
with caplog.at_level(logging.WARNING):
results = [
await jev.decide(f"state {index}", {"urgent": URGENT}, scope="exec-1")
for index in range(4)
]
other = await jev.decide("state", {"urgent": URGENT}, scope="exec-2")
assert [result is not None for result in results] == [True, True, False, False]
assert other is not None # another Execution has its own budget
assert len(service.requests) == 3
assert caplog.text.count("budget reached for exec-1") == 1
async def test_each_execution_has_a_budget_of_tokens():
service = Service(default=ok("urgent", tokens=600))
jev = decider(service, budget_tokens=1000, burst=50)
results = [
await jev.decide(f"state {index}", {"urgent": URGENT}, scope="exec-1")
for index in range(3)
]
assert [result is not None for result in results] == [True, True, False]
async def test_the_key_is_never_logged(caplog):
service = Service(httpx2.Response(500), httpx2.Response(401))
jev = decider(service, retries=0)
with caplog.at_level(logging.DEBUG):
await jev.decide("a", {"urgent": URGENT})
await jev.decide("b", {"urgent": URGENT})
assert "tsk_secret" not in caplog.text
# --- what is recorded, and running without Jev -------------------------------------------
async def test_every_decision_is_recorded_for_the_evaluation_report():
service = Service(ok("urgent"), default=httpx2.Response(400))
jev = decider(service)
await jev.decide("first", {"urgent": URGENT}, scope="exec-1")
await jev.decide("first", {"urgent": URGENT}, scope="exec-1")
await jev.decide("second", {"urgent": URGENT}, scope="exec-1")
assert round(jev.records[0].confidence["urgent"], 2) == 0.94
assert jev.summary() == {
"decisions": 3,
"outcomes": {"answered": 1, "cached": 1, "failed": 1},
"fallbacks": 1,
"rate_limited": 0,
"seconds": round(jev.records[0].seconds, 3),
"input_tokens": 40,
}
async def test_without_a_key_every_caller_takes_its_fallback():
unset = open_decider(Settings(_env_file=None, TYPESAFE_API_KEY=None))
assert isinstance(unset, NoDecider)
assert await unset.decide("state", {"urgent": URGENT}) is None
async def test_with_a_key_the_client_takes_its_limits_from_settings():
configured = open_decider(
Settings(
_env_file=None,
TYPESAFE_API_KEY="tsk_x",
TYPESAFE_BASE_URL="https://jev.internal",
JEV_REQUESTS_PER_SECOND=2,
JEV_BUDGET_CALLS=40,
)
)
assert isinstance(configured, JevDecider)
assert configured.limits.requests_per_second == 2
assert configured.limits.budget_calls == 40
assert configured.limits.max_tokens == 48_000
await configured.aclose()

View File

@@ -1,5 +1,6 @@
"""Local tools: Mermaid check and Terraform fmt/validate/plan in a scratch directory."""
import json
import shutil
from pathlib import Path
@@ -140,6 +141,110 @@ async def test_plan_never_runs_apply_and_refuses_unsafe_variables(monkeypatch):
assert "-var=region=europe-west2" in plan
# --- schema lookup -----------------------------------------------------------------------
SCHEMA = {
"provider_schemas": {
"registry.terraform.io/example/cloud": {
"resource_schemas": {
"cloud_service": {
"block": {
"attributes": {
"name": {"type": "string", "required": True},
"labels": {"type": ["map", "string"], "optional": True},
"uri": {"type": "string", "computed": True},
},
"block_types": {
"template": {
"nesting_mode": "list",
"max_items": 1,
"block": {
"attributes": {
"cpu_boost": {
"type": "bool",
"optional": True,
"description": "Boost CPU at start.",
}
}
},
}
},
}
},
"cloud_job": {"block": {"attributes": {"name": {"type": "string"}}}},
}
}
}
}
PIN = {"cloud": {"source": "example/cloud", "version": "1.2.3"}}
def schema_runner(monkeypatch) -> tuple[TerraformRunner, list[list[str]]]:
commands: list[list[str]] = []
async def fake_run(self, args, cwd, env=None):
commands.append(list(args))
if args[1] == "init":
assert (
'cloud = { source = "example/cloud", version = "1.2.3" }'
in (Path(cwd) / "versions.tf").read_text()
)
return 0, "ok"
return 0, json.dumps(SCHEMA)
monkeypatch.setattr(TerraformRunner, "_run", fake_run)
return TerraformRunner(providers=PIN), commands
async def test_schema_lists_what_code_may_set_and_outlines_the_blocks(monkeypatch):
runner, _ = schema_runner(monkeypatch)
result = await runner.schema("cloud_service")
assert result.message.startswith("cloud_service (example/cloud 1.2.3)")
assert " name (string, required)" in result.message
assert " labels (map of string, optional)" in result.message
assert "uri" not in result.message # read-only: code cannot set it
assert " template { } (list, at most 1): cpu_boost" in result.message
async def test_schema_gives_one_block_in_full(monkeypatch):
runner, _ = schema_runner(monkeypatch)
result = await runner.schema("cloud_service", block="template")
missing = await runner.schema("cloud_service", block="template.containers")
assert " cpu_boost (bool, optional): Boost CPU at start." in result.message
assert "'containers' is not a block there" in missing.message
async def test_schema_says_where_a_setting_sits_or_that_it_cannot_be_set(monkeypatch):
runner, _ = schema_runner(monkeypatch)
found = await runner.schema("cloud_service", find="cpu_boost")
absent = await runner.schema("cloud_job", find="cpu_boost")
assert "template.cpu_boost (bool, optional)" in found.message
assert "It cannot be set on this resource type." in absent.message
async def test_schema_suggests_close_resource_types_and_is_loaded_once(monkeypatch):
runner, commands = schema_runner(monkeypatch)
unknown = await runner.schema("cloud_servce")
await runner.schema("cloud_job")
assert "Close matches: cloud_service" in unknown.message
assert [command[1] for command in commands] == ["init", "providers"]
assert "apply" not in [arg for command in commands for arg in command]
async def test_schema_is_unavailable_without_a_pinned_provider():
result = await TerraformRunner().schema("cloud_service")
assert "pins no Terraform provider" in result.message
# --- registry ------------------------------------------------------------------------------
@@ -155,6 +260,7 @@ def test_local_tools_are_built_by_name_with_their_tier():
"terraform_fmt",
"terraform_validate",
"terraform_plan",
"terraform_schema",
}
with pytest.raises(KeyError, match="terraform_apply"):
local_tools(["terraform_apply"], TerraformRunner())