This commit is contained in:
417
tests/tools/test_jev.py
Normal file
417
tests/tools/test_jev.py
Normal file
@@ -0,0 +1,417 @@
|
||||
"""The Jev decision client: typed answers within strict limits, and a fallback always.
|
||||
|
||||
Every test runs the real TypeSafe SDK against a stand-in transport; nothing here calls
|
||||
the live service.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
import httpx2
|
||||
|
||||
from app.adk.runtime import Answer, NoDecider, Question
|
||||
from app.agent import open_decider
|
||||
from app.config import Settings
|
||||
from app.tools.jev import JevDecider, JevLimits, estimate_tokens
|
||||
|
||||
URGENT = Question("noul", "Does this need attention right now?")
|
||||
ROUTE = Question(
|
||||
"choice", "What comes next?", ("revise_code", "revise_graph", "proceed")
|
||||
)
|
||||
SEVERITY = Question("score", "How much does it matter?", ("low", "medium", "high"))
|
||||
ALL = {"urgent": URGENT, "route": ROUTE, "severity": SEVERITY}
|
||||
|
||||
ANSWERS = {
|
||||
"urgent": {"type": "noul", "noul": 0.97},
|
||||
"route": {
|
||||
"type": "choice",
|
||||
"choice": "revise_code",
|
||||
"confidence": 0.81,
|
||||
"probabilities": {"revise_code": 0.81, "revise_graph": 0.12, "proceed": 0.07},
|
||||
},
|
||||
"severity": {
|
||||
"type": "score",
|
||||
"score": 1.6,
|
||||
"confidence": 0.7,
|
||||
"legend": {"0": "low", "1": "medium", "2": "high"},
|
||||
"probabilities": {"0": 0.1, "1": 0.2, "2": 0.7},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class Clock:
|
||||
"""Time that moves only when something sleeps."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.now = 1000.0
|
||||
self.slept: list[float] = []
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.now
|
||||
|
||||
async def sleep(self, seconds: float) -> None:
|
||||
self.slept.append(seconds)
|
||||
self.now += seconds
|
||||
|
||||
|
||||
def ok(*names: str, tokens: int = 40) -> httpx2.Response:
|
||||
answers = {name: ANSWERS[name] for name in names} if names else ANSWERS
|
||||
return httpx2.Response(
|
||||
200,
|
||||
json={
|
||||
"model": "jev-latest",
|
||||
"answers": answers,
|
||||
"usage": {"input_tokens": tokens, "output_tokens": 0},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class Service:
|
||||
"""A stand-in Jev: replies from a queue, then with `default`; records requests.
|
||||
|
||||
A reply answers only the questions the request asked, as the service does.
|
||||
"""
|
||||
|
||||
def __init__(self, *replies, default=None) -> None:
|
||||
self.replies = list(replies)
|
||||
self.default = default
|
||||
self.requests: list[httpx2.Request] = []
|
||||
|
||||
def __call__(self, request: httpx2.Request) -> httpx2.Response:
|
||||
self.requests.append(request)
|
||||
reply = self.replies.pop(0) if self.replies else self.default
|
||||
if reply is None:
|
||||
return ok(*json.loads(request.content)["questions"])
|
||||
if isinstance(reply, Exception):
|
||||
raise reply
|
||||
return reply
|
||||
|
||||
@property
|
||||
def bodies(self) -> list[dict]:
|
||||
return [json.loads(request.content) for request in self.requests]
|
||||
|
||||
|
||||
def decider(service: Service, clock: Clock | None = None, **limits) -> JevDecider:
|
||||
clock = clock or Clock()
|
||||
limits.setdefault("backoff_base", 0.0) # no real waiting between SDK retries
|
||||
return JevDecider(
|
||||
api_key="tsk_secret",
|
||||
base_url="https://jev.example",
|
||||
limits=JevLimits(**limits),
|
||||
transport=httpx2.MockTransport(service),
|
||||
clock=clock,
|
||||
sleep=clock.sleep,
|
||||
)
|
||||
|
||||
|
||||
# --- the request and the typed answers -------------------------------------------------
|
||||
|
||||
|
||||
async def test_one_request_carries_the_state_and_every_question():
|
||||
service = Service()
|
||||
jev = decider(service)
|
||||
|
||||
answers = await jev.decide("K-30 failed twice.", ALL, scope="exec-1")
|
||||
|
||||
[request] = service.requests
|
||||
assert request.method == "POST"
|
||||
assert str(request.url) == "https://jev.example/v1/systemone"
|
||||
assert request.headers["Authorization"] == "Bearer tsk_secret"
|
||||
body = service.bodies[0]
|
||||
assert body["state"] == "K-30 failed twice."
|
||||
assert body["questions"] == {
|
||||
"urgent": {"type": "noul", "instructions": URGENT.instructions},
|
||||
"route": {
|
||||
"type": "choice",
|
||||
"instructions": ROUTE.instructions,
|
||||
"criteria": {"revise_code": None, "revise_graph": None, "proceed": None},
|
||||
},
|
||||
"severity": {
|
||||
"type": "score",
|
||||
"instructions": SEVERITY.instructions,
|
||||
"criteria": ["low", "medium", "high"],
|
||||
},
|
||||
}
|
||||
assert answers["route"] == Answer(
|
||||
"revise_code",
|
||||
0.81,
|
||||
{"revise_code": 0.81, "revise_graph": 0.12, "proceed": 0.07},
|
||||
)
|
||||
assert answers["severity"] == Answer(1.6, 0.7, {"0": 0.1, "1": 0.2, "2": 0.7})
|
||||
# A yes-or-no answer is a probability; its confidence is its distance from even.
|
||||
assert answers["urgent"].value == 0.97
|
||||
assert round(answers["urgent"].confidence, 2) == 0.94
|
||||
assert answers["urgent"].is_yes() and not Answer(0.2, 0.6).is_yes()
|
||||
|
||||
|
||||
async def test_a_choice_outside_the_options_or_a_missing_answer_is_no_decision():
|
||||
invented = httpx2.Response(
|
||||
200,
|
||||
json={
|
||||
"model": "jev-latest",
|
||||
"usage": {"input_tokens": 1, "output_tokens": 0},
|
||||
"answers": {
|
||||
"route": {
|
||||
"type": "choice",
|
||||
"choice": "delete_everything",
|
||||
"confidence": 0.99,
|
||||
"probabilities": {},
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert await decider(Service(invented)).decide("s", {"route": ROUTE}) is None
|
||||
assert await decider(Service(ok("urgent"))).decide("s", ALL) is None
|
||||
assert (
|
||||
await decider(Service(httpx2.Response(200, text="not json"))).decide(
|
||||
"s", {"urgent": URGENT}
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
async def test_no_questions_needs_no_request():
|
||||
service = Service()
|
||||
|
||||
assert await decider(service).decide("state", {}) == {}
|
||||
assert service.requests == []
|
||||
|
||||
|
||||
# --- size --------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_request_over_the_size_limit_is_refused_locally_never_truncated():
|
||||
service = Service()
|
||||
jev = decider(service, max_tokens=100, max_question_tokens=100)
|
||||
state = "x" * 600
|
||||
|
||||
assert estimate_tokens(state) > 100
|
||||
assert await jev.decide(state, {"urgent": URGENT}) is None
|
||||
assert service.requests == []
|
||||
assert jev.records[-1].outcome == "too_large"
|
||||
|
||||
|
||||
# --- timeouts and retries (the SDK's, configured here) ------------------------------------
|
||||
|
||||
|
||||
async def test_temporary_failures_are_retried_then_answered():
|
||||
service = Service(httpx2.Response(503), httpx2.ConnectTimeout("slow"))
|
||||
jev = decider(service)
|
||||
|
||||
assert await jev.decide("s", {"urgent": URGENT}) is not None
|
||||
assert len(service.requests) == 3
|
||||
|
||||
|
||||
async def test_a_rate_limit_that_outlasts_the_retries_lowers_the_clients_own_rate():
|
||||
service = Service(default=httpx2.Response(429))
|
||||
jev = decider(service, retries=1)
|
||||
|
||||
assert await jev.decide("s", {"urgent": URGENT}) is None
|
||||
assert len(service.requests) == 2
|
||||
assert jev.records[-1].rate_limited
|
||||
assert jev._rate == JevLimits().requests_per_second / 2
|
||||
|
||||
|
||||
async def test_a_rejected_request_is_not_retried():
|
||||
service = Service(default=httpx2.Response(400, json={"error": "bad question"}))
|
||||
jev = decider(service)
|
||||
|
||||
assert await jev.decide("s", {"urgent": URGENT}) is None
|
||||
assert len(service.requests) == 1
|
||||
|
||||
|
||||
async def test_retries_stop_at_the_budget_and_the_caller_falls_back():
|
||||
service = Service(default=httpx2.Response(503))
|
||||
jev = decider(service, retries=2)
|
||||
|
||||
assert await jev.decide("s", {"urgent": URGENT}) is None
|
||||
assert len(service.requests) == 3
|
||||
assert jev.records[-1].outcome == "failed"
|
||||
|
||||
|
||||
# --- rate and concurrency ----------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_calls_beyond_the_burst_wait_for_the_bucket_to_refill():
|
||||
clock = Clock()
|
||||
service = Service()
|
||||
jev = decider(service, clock, burst=2, requests_per_second=4)
|
||||
|
||||
for index in range(3):
|
||||
await jev.decide(f"state {index}", {"urgent": URGENT})
|
||||
|
||||
assert len(service.requests) == 3
|
||||
assert clock.slept == [0.25] # one token at four a second
|
||||
|
||||
|
||||
async def test_calls_in_flight_never_exceed_the_cap():
|
||||
in_flight = peak = 0
|
||||
|
||||
async def slow(request: httpx2.Request) -> httpx2.Response:
|
||||
nonlocal in_flight, peak
|
||||
in_flight += 1
|
||||
peak = max(peak, in_flight)
|
||||
await asyncio.sleep(0.01)
|
||||
in_flight -= 1
|
||||
return ok("urgent")
|
||||
|
||||
jev = JevDecider(
|
||||
api_key="k",
|
||||
base_url="https://jev.example",
|
||||
limits=JevLimits(max_in_flight=2, burst=50),
|
||||
transport=httpx2.MockTransport(slow),
|
||||
)
|
||||
|
||||
await asyncio.gather(
|
||||
*(jev.decide(f"state {index}", {"urgent": URGENT}) for index in range(8))
|
||||
)
|
||||
|
||||
assert peak == 2
|
||||
|
||||
|
||||
# --- circuit breaker ---------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_repeated_failures_open_the_circuit_and_one_probe_closes_it():
|
||||
clock = Clock()
|
||||
service = Service(default=httpx2.Response(503))
|
||||
jev = decider(
|
||||
service, clock, retries=0, breaker_failures=3, breaker_cooldown=60, burst=50
|
||||
)
|
||||
|
||||
for index in range(3):
|
||||
assert await jev.decide(f"s{index}", {"urgent": URGENT}) is None
|
||||
sent = len(service.requests)
|
||||
|
||||
# Open: the service is not called at all.
|
||||
assert await jev.decide("s-open", {"urgent": URGENT}) is None
|
||||
assert len(service.requests) == sent
|
||||
assert jev.records[-1].outcome == "breaker_open"
|
||||
|
||||
# After the cool-down a single call probes; its success closes the circuit.
|
||||
clock.now += 61
|
||||
service.default = None
|
||||
assert await jev.decide("s-probe", {"urgent": URGENT}) is not None
|
||||
assert await jev.decide("s-after", {"urgent": URGENT}) is not None
|
||||
|
||||
|
||||
async def test_a_failed_probe_reopens_the_circuit():
|
||||
clock = Clock()
|
||||
service = Service(default=httpx2.Response(503))
|
||||
jev = decider(
|
||||
service, clock, retries=0, breaker_failures=2, breaker_cooldown=60, burst=50
|
||||
)
|
||||
for index in range(2):
|
||||
await jev.decide(f"s{index}", {"urgent": URGENT})
|
||||
|
||||
clock.now += 61
|
||||
await jev.decide("probe", {"urgent": URGENT})
|
||||
sent = len(service.requests)
|
||||
|
||||
assert await jev.decide("again", {"urgent": URGENT}) is None
|
||||
assert len(service.requests) == sent
|
||||
|
||||
|
||||
# --- cache and budget --------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_the_same_question_about_the_same_state_is_answered_from_memory():
|
||||
service = Service()
|
||||
jev = decider(service)
|
||||
|
||||
first = await jev.decide("unchanged", {"route": ROUTE})
|
||||
second = await jev.decide("unchanged", {"route": ROUTE})
|
||||
await jev.decide("changed", {"route": ROUTE})
|
||||
|
||||
assert first == second
|
||||
assert len(service.requests) == 2
|
||||
assert [r.outcome for r in jev.records] == ["answered", "cached", "answered"]
|
||||
|
||||
|
||||
async def test_each_execution_has_a_budget_of_calls_and_then_falls_back(caplog):
|
||||
service = Service()
|
||||
jev = decider(service, budget_calls=2, burst=50)
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
results = [
|
||||
await jev.decide(f"state {index}", {"urgent": URGENT}, scope="exec-1")
|
||||
for index in range(4)
|
||||
]
|
||||
other = await jev.decide("state", {"urgent": URGENT}, scope="exec-2")
|
||||
|
||||
assert [result is not None for result in results] == [True, True, False, False]
|
||||
assert other is not None # another Execution has its own budget
|
||||
assert len(service.requests) == 3
|
||||
assert caplog.text.count("budget reached for exec-1") == 1
|
||||
|
||||
|
||||
async def test_each_execution_has_a_budget_of_tokens():
|
||||
service = Service(default=ok("urgent", tokens=600))
|
||||
jev = decider(service, budget_tokens=1000, burst=50)
|
||||
|
||||
results = [
|
||||
await jev.decide(f"state {index}", {"urgent": URGENT}, scope="exec-1")
|
||||
for index in range(3)
|
||||
]
|
||||
|
||||
assert [result is not None for result in results] == [True, True, False]
|
||||
|
||||
|
||||
async def test_the_key_is_never_logged(caplog):
|
||||
service = Service(httpx2.Response(500), httpx2.Response(401))
|
||||
jev = decider(service, retries=0)
|
||||
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
await jev.decide("a", {"urgent": URGENT})
|
||||
await jev.decide("b", {"urgent": URGENT})
|
||||
|
||||
assert "tsk_secret" not in caplog.text
|
||||
|
||||
|
||||
# --- what is recorded, and running without Jev -------------------------------------------
|
||||
|
||||
|
||||
async def test_every_decision_is_recorded_for_the_evaluation_report():
|
||||
service = Service(ok("urgent"), default=httpx2.Response(400))
|
||||
jev = decider(service)
|
||||
|
||||
await jev.decide("first", {"urgent": URGENT}, scope="exec-1")
|
||||
await jev.decide("first", {"urgent": URGENT}, scope="exec-1")
|
||||
await jev.decide("second", {"urgent": URGENT}, scope="exec-1")
|
||||
|
||||
assert round(jev.records[0].confidence["urgent"], 2) == 0.94
|
||||
assert jev.summary() == {
|
||||
"decisions": 3,
|
||||
"outcomes": {"answered": 1, "cached": 1, "failed": 1},
|
||||
"fallbacks": 1,
|
||||
"rate_limited": 0,
|
||||
"seconds": round(jev.records[0].seconds, 3),
|
||||
"input_tokens": 40,
|
||||
}
|
||||
|
||||
|
||||
async def test_without_a_key_every_caller_takes_its_fallback():
|
||||
unset = open_decider(Settings(_env_file=None, TYPESAFE_API_KEY=None))
|
||||
|
||||
assert isinstance(unset, NoDecider)
|
||||
assert await unset.decide("state", {"urgent": URGENT}) is None
|
||||
|
||||
|
||||
async def test_with_a_key_the_client_takes_its_limits_from_settings():
|
||||
configured = open_decider(
|
||||
Settings(
|
||||
_env_file=None,
|
||||
TYPESAFE_API_KEY="tsk_x",
|
||||
TYPESAFE_BASE_URL="https://jev.internal",
|
||||
JEV_REQUESTS_PER_SECOND=2,
|
||||
JEV_BUDGET_CALLS=40,
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(configured, JevDecider)
|
||||
assert configured.limits.requests_per_second == 2
|
||||
assert configured.limits.budget_calls == 40
|
||||
assert configured.limits.max_tokens == 48_000
|
||||
await configured.aclose()
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Local tools: Mermaid check and Terraform fmt/validate/plan in a scratch directory."""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
@@ -140,6 +141,110 @@ async def test_plan_never_runs_apply_and_refuses_unsafe_variables(monkeypatch):
|
||||
assert "-var=region=europe-west2" in plan
|
||||
|
||||
|
||||
# --- schema lookup -----------------------------------------------------------------------
|
||||
|
||||
SCHEMA = {
|
||||
"provider_schemas": {
|
||||
"registry.terraform.io/example/cloud": {
|
||||
"resource_schemas": {
|
||||
"cloud_service": {
|
||||
"block": {
|
||||
"attributes": {
|
||||
"name": {"type": "string", "required": True},
|
||||
"labels": {"type": ["map", "string"], "optional": True},
|
||||
"uri": {"type": "string", "computed": True},
|
||||
},
|
||||
"block_types": {
|
||||
"template": {
|
||||
"nesting_mode": "list",
|
||||
"max_items": 1,
|
||||
"block": {
|
||||
"attributes": {
|
||||
"cpu_boost": {
|
||||
"type": "bool",
|
||||
"optional": True,
|
||||
"description": "Boost CPU at start.",
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
"cloud_job": {"block": {"attributes": {"name": {"type": "string"}}}},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
PIN = {"cloud": {"source": "example/cloud", "version": "1.2.3"}}
|
||||
|
||||
|
||||
def schema_runner(monkeypatch) -> tuple[TerraformRunner, list[list[str]]]:
|
||||
commands: list[list[str]] = []
|
||||
|
||||
async def fake_run(self, args, cwd, env=None):
|
||||
commands.append(list(args))
|
||||
if args[1] == "init":
|
||||
assert (
|
||||
'cloud = { source = "example/cloud", version = "1.2.3" }'
|
||||
in (Path(cwd) / "versions.tf").read_text()
|
||||
)
|
||||
return 0, "ok"
|
||||
return 0, json.dumps(SCHEMA)
|
||||
|
||||
monkeypatch.setattr(TerraformRunner, "_run", fake_run)
|
||||
return TerraformRunner(providers=PIN), commands
|
||||
|
||||
|
||||
async def test_schema_lists_what_code_may_set_and_outlines_the_blocks(monkeypatch):
|
||||
runner, _ = schema_runner(monkeypatch)
|
||||
|
||||
result = await runner.schema("cloud_service")
|
||||
|
||||
assert result.message.startswith("cloud_service (example/cloud 1.2.3)")
|
||||
assert " name (string, required)" in result.message
|
||||
assert " labels (map of string, optional)" in result.message
|
||||
assert "uri" not in result.message # read-only: code cannot set it
|
||||
assert " template { } (list, at most 1): cpu_boost" in result.message
|
||||
|
||||
|
||||
async def test_schema_gives_one_block_in_full(monkeypatch):
|
||||
runner, _ = schema_runner(monkeypatch)
|
||||
|
||||
result = await runner.schema("cloud_service", block="template")
|
||||
missing = await runner.schema("cloud_service", block="template.containers")
|
||||
|
||||
assert " cpu_boost (bool, optional): Boost CPU at start." in result.message
|
||||
assert "'containers' is not a block there" in missing.message
|
||||
|
||||
|
||||
async def test_schema_says_where_a_setting_sits_or_that_it_cannot_be_set(monkeypatch):
|
||||
runner, _ = schema_runner(monkeypatch)
|
||||
|
||||
found = await runner.schema("cloud_service", find="cpu_boost")
|
||||
absent = await runner.schema("cloud_job", find="cpu_boost")
|
||||
|
||||
assert "template.cpu_boost (bool, optional)" in found.message
|
||||
assert "It cannot be set on this resource type." in absent.message
|
||||
|
||||
|
||||
async def test_schema_suggests_close_resource_types_and_is_loaded_once(monkeypatch):
|
||||
runner, commands = schema_runner(monkeypatch)
|
||||
|
||||
unknown = await runner.schema("cloud_servce")
|
||||
await runner.schema("cloud_job")
|
||||
|
||||
assert "Close matches: cloud_service" in unknown.message
|
||||
assert [command[1] for command in commands] == ["init", "providers"]
|
||||
assert "apply" not in [arg for command in commands for arg in command]
|
||||
|
||||
|
||||
async def test_schema_is_unavailable_without_a_pinned_provider():
|
||||
result = await TerraformRunner().schema("cloud_service")
|
||||
|
||||
assert "pins no Terraform provider" in result.message
|
||||
|
||||
|
||||
# --- registry ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -155,6 +260,7 @@ def test_local_tools_are_built_by_name_with_their_tier():
|
||||
"terraform_fmt",
|
||||
"terraform_validate",
|
||||
"terraform_plan",
|
||||
"terraform_schema",
|
||||
}
|
||||
with pytest.raises(KeyError, match="terraform_apply"):
|
||||
local_tools(["terraform_apply"], TerraformRunner())
|
||||
|
||||
Reference in New Issue
Block a user