"""The Jev decision client: typed answers within strict limits, and a fallback always. Every test runs the real TypeSafe SDK against a stand-in transport; nothing here calls the live service. """ import asyncio import json import logging import httpx2 from app.adk.runtime import Answer, NoDecider, Question from app.agent import open_decider from app.config import Settings from app.tools.jev import JevDecider, JevLimits, estimate_tokens URGENT = Question("noul", "Does this need attention right now?") ROUTE = Question( "choice", "What comes next?", ("revise_code", "revise_graph", "proceed") ) SEVERITY = Question("score", "How much does it matter?", ("low", "medium", "high")) ALL = {"urgent": URGENT, "route": ROUTE, "severity": SEVERITY} ANSWERS = { "urgent": {"type": "noul", "noul": 0.97}, "route": { "type": "choice", "choice": "revise_code", "confidence": 0.81, "probabilities": {"revise_code": 0.81, "revise_graph": 0.12, "proceed": 0.07}, }, "severity": { "type": "score", "score": 1.6, "confidence": 0.7, "legend": {"0": "low", "1": "medium", "2": "high"}, "probabilities": {"0": 0.1, "1": 0.2, "2": 0.7}, }, } class Clock: """Time that moves only when something sleeps.""" def __init__(self) -> None: self.now = 1000.0 self.slept: list[float] = [] def __call__(self) -> float: return self.now async def sleep(self, seconds: float) -> None: self.slept.append(seconds) self.now += seconds def ok(*names: str, tokens: int = 40) -> httpx2.Response: answers = {name: ANSWERS[name] for name in names} if names else ANSWERS return httpx2.Response( 200, json={ "model": "jev-latest", "answers": answers, "usage": {"input_tokens": tokens, "output_tokens": 0}, }, ) class Service: """A stand-in Jev: replies from a queue, then with `default`; records requests. A reply answers only the questions the request asked, as the service does. """ def __init__(self, *replies, default=None) -> None: self.replies = list(replies) self.default = default self.requests: list[httpx2.Request] = [] def __call__(self, request: httpx2.Request) -> httpx2.Response: self.requests.append(request) reply = self.replies.pop(0) if self.replies else self.default if reply is None: return ok(*json.loads(request.content)["questions"]) if isinstance(reply, Exception): raise reply return reply @property def bodies(self) -> list[dict]: return [json.loads(request.content) for request in self.requests] def decider(service: Service, clock: Clock | None = None, **limits) -> JevDecider: clock = clock or Clock() limits.setdefault("backoff_base", 0.0) # no real waiting between SDK retries return JevDecider( api_key="tsk_secret", base_url="https://jev.example", limits=JevLimits(**limits), transport=httpx2.MockTransport(service), clock=clock, sleep=clock.sleep, ) # --- the request and the typed answers ------------------------------------------------- async def test_one_request_carries_the_state_and_every_question(): service = Service() jev = decider(service) answers = await jev.decide("K-30 failed twice.", ALL, scope="exec-1") [request] = service.requests assert request.method == "POST" assert str(request.url) == "https://jev.example/v1/systemone" assert request.headers["Authorization"] == "Bearer tsk_secret" body = service.bodies[0] assert body["state"] == "K-30 failed twice." assert body["questions"] == { "urgent": {"type": "noul", "instructions": URGENT.instructions}, "route": { "type": "choice", "instructions": ROUTE.instructions, "criteria": {"revise_code": None, "revise_graph": None, "proceed": None}, }, "severity": { "type": "score", "instructions": SEVERITY.instructions, "criteria": ["low", "medium", "high"], }, } assert answers["route"] == Answer( "revise_code", 0.81, {"revise_code": 0.81, "revise_graph": 0.12, "proceed": 0.07}, ) assert answers["severity"] == Answer(1.6, 0.7, {"0": 0.1, "1": 0.2, "2": 0.7}) # A yes-or-no answer is a probability; its confidence is its distance from even. assert answers["urgent"].value == 0.97 assert round(answers["urgent"].confidence, 2) == 0.94 assert answers["urgent"].is_yes() and not Answer(0.2, 0.6).is_yes() async def test_a_choice_outside_the_options_or_a_missing_answer_is_no_decision(): invented = httpx2.Response( 200, json={ "model": "jev-latest", "usage": {"input_tokens": 1, "output_tokens": 0}, "answers": { "route": { "type": "choice", "choice": "delete_everything", "confidence": 0.99, "probabilities": {}, } }, }, ) assert await decider(Service(invented)).decide("s", {"route": ROUTE}) is None assert await decider(Service(ok("urgent"))).decide("s", ALL) is None assert ( await decider(Service(httpx2.Response(200, text="not json"))).decide( "s", {"urgent": URGENT} ) is None ) async def test_no_questions_needs_no_request(): service = Service() assert await decider(service).decide("state", {}) == {} assert service.requests == [] # --- size -------------------------------------------------------------------------------- async def test_a_request_over_the_size_limit_is_refused_locally_never_truncated(): service = Service() jev = decider(service, max_tokens=100, max_question_tokens=100) state = "x" * 600 assert estimate_tokens(state) > 100 assert await jev.decide(state, {"urgent": URGENT}) is None assert service.requests == [] assert jev.records[-1].outcome == "too_large" # --- timeouts and retries (the SDK's, configured here) ------------------------------------ async def test_temporary_failures_are_retried_then_answered(): service = Service(httpx2.Response(503), httpx2.ConnectTimeout("slow")) jev = decider(service) assert await jev.decide("s", {"urgent": URGENT}) is not None assert len(service.requests) == 3 async def test_a_rate_limit_that_outlasts_the_retries_lowers_the_clients_own_rate(): service = Service(default=httpx2.Response(429)) jev = decider(service, retries=1) assert await jev.decide("s", {"urgent": URGENT}) is None assert len(service.requests) == 2 assert jev.records[-1].rate_limited assert jev._rate == JevLimits().requests_per_second / 2 async def test_a_rejected_request_is_not_retried(): service = Service(default=httpx2.Response(400, json={"error": "bad question"})) jev = decider(service) assert await jev.decide("s", {"urgent": URGENT}) is None assert len(service.requests) == 1 async def test_retries_stop_at_the_budget_and_the_caller_falls_back(): service = Service(default=httpx2.Response(503)) jev = decider(service, retries=2) assert await jev.decide("s", {"urgent": URGENT}) is None assert len(service.requests) == 3 assert jev.records[-1].outcome == "failed" # --- rate and concurrency ---------------------------------------------------------------- async def test_calls_beyond_the_burst_wait_for_the_bucket_to_refill(): clock = Clock() service = Service() jev = decider(service, clock, burst=2, requests_per_second=4) for index in range(3): await jev.decide(f"state {index}", {"urgent": URGENT}) assert len(service.requests) == 3 assert clock.slept == [0.25] # one token at four a second async def test_calls_in_flight_never_exceed_the_cap(): in_flight = peak = 0 async def slow(request: httpx2.Request) -> httpx2.Response: nonlocal in_flight, peak in_flight += 1 peak = max(peak, in_flight) await asyncio.sleep(0.01) in_flight -= 1 return ok("urgent") jev = JevDecider( api_key="k", base_url="https://jev.example", limits=JevLimits(max_in_flight=2, burst=50), transport=httpx2.MockTransport(slow), ) await asyncio.gather( *(jev.decide(f"state {index}", {"urgent": URGENT}) for index in range(8)) ) assert peak == 2 # --- circuit breaker --------------------------------------------------------------------- async def test_repeated_failures_open_the_circuit_and_one_probe_closes_it(): clock = Clock() service = Service(default=httpx2.Response(503)) jev = decider( service, clock, retries=0, breaker_failures=3, breaker_cooldown=60, burst=50 ) for index in range(3): assert await jev.decide(f"s{index}", {"urgent": URGENT}) is None sent = len(service.requests) # Open: the service is not called at all. assert await jev.decide("s-open", {"urgent": URGENT}) is None assert len(service.requests) == sent assert jev.records[-1].outcome == "breaker_open" # After the cool-down a single call probes; its success closes the circuit. clock.now += 61 service.default = None assert await jev.decide("s-probe", {"urgent": URGENT}) is not None assert await jev.decide("s-after", {"urgent": URGENT}) is not None async def test_a_failed_probe_reopens_the_circuit(): clock = Clock() service = Service(default=httpx2.Response(503)) jev = decider( service, clock, retries=0, breaker_failures=2, breaker_cooldown=60, burst=50 ) for index in range(2): await jev.decide(f"s{index}", {"urgent": URGENT}) clock.now += 61 await jev.decide("probe", {"urgent": URGENT}) sent = len(service.requests) assert await jev.decide("again", {"urgent": URGENT}) is None assert len(service.requests) == sent # --- cache and budget -------------------------------------------------------------------- async def test_the_same_question_about_the_same_state_is_answered_from_memory(): service = Service() jev = decider(service) first = await jev.decide("unchanged", {"route": ROUTE}) second = await jev.decide("unchanged", {"route": ROUTE}) await jev.decide("changed", {"route": ROUTE}) assert first == second assert len(service.requests) == 2 assert [r.outcome for r in jev.records] == ["answered", "cached", "answered"] async def test_each_execution_has_a_budget_of_calls_and_then_falls_back(caplog): service = Service() jev = decider(service, budget_calls=2, burst=50) with caplog.at_level(logging.WARNING): results = [ await jev.decide(f"state {index}", {"urgent": URGENT}, scope="exec-1") for index in range(4) ] other = await jev.decide("state", {"urgent": URGENT}, scope="exec-2") assert [result is not None for result in results] == [True, True, False, False] assert other is not None # another Execution has its own budget assert len(service.requests) == 3 assert caplog.text.count("budget reached for exec-1") == 1 async def test_each_execution_has_a_budget_of_tokens(): service = Service(default=ok("urgent", tokens=600)) jev = decider(service, budget_tokens=1000, burst=50) results = [ await jev.decide(f"state {index}", {"urgent": URGENT}, scope="exec-1") for index in range(3) ] assert [result is not None for result in results] == [True, True, False] async def test_the_key_is_never_logged(caplog): service = Service(httpx2.Response(500), httpx2.Response(401)) jev = decider(service, retries=0) with caplog.at_level(logging.DEBUG): await jev.decide("a", {"urgent": URGENT}) await jev.decide("b", {"urgent": URGENT}) assert "tsk_secret" not in caplog.text # --- what is recorded, and running without Jev ------------------------------------------- async def test_every_decision_is_recorded_for_the_evaluation_report(): service = Service(ok("urgent"), default=httpx2.Response(400)) jev = decider(service) await jev.decide("first", {"urgent": URGENT}, scope="exec-1") await jev.decide("first", {"urgent": URGENT}, scope="exec-1") await jev.decide("second", {"urgent": URGENT}, scope="exec-1") assert round(jev.records[0].confidence["urgent"], 2) == 0.94 assert jev.summary() == { "decisions": 3, "outcomes": {"answered": 1, "cached": 1, "failed": 1}, "fallbacks": 1, "rate_limited": 0, "seconds": round(jev.records[0].seconds, 3), "input_tokens": 40, } async def test_without_a_key_every_caller_takes_its_fallback(): unset = open_decider(Settings(_env_file=None, TYPESAFE_API_KEY=None)) assert isinstance(unset, NoDecider) assert await unset.decide("state", {"urgent": URGENT}) is None async def test_with_a_key_the_client_takes_its_limits_from_settings(): configured = open_decider( Settings( _env_file=None, TYPESAFE_API_KEY="tsk_x", TYPESAFE_BASE_URL="https://jev.internal", JEV_REQUESTS_PER_SECOND=2, JEV_BUDGET_CALLS=40, ) ) assert isinstance(configured, JevDecider) assert configured.limits.requests_per_second == 2 assert configured.limits.budget_calls == 40 assert configured.limits.max_tokens == 48_000 await configured.aclose()