"""The Jev decision client: typed answers to closed questions, within strict limits. Jev is TypeSafe AI's decision model. This module is the only one that speaks to it, through the vendor's Python SDK (`typesafe-sdk`), which owns the request shape, the timeout and the retries. Everything else here is ours: the service's limits are not published, so the client assumes they are strict and never lets Jev hold up an Execution. Any refusal, failure or timeout returns None and the caller takes its fallback. """ from __future__ import annotations import asyncio import hashlib import json import logging import time from collections import OrderedDict, deque from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass, field from typing import Any from typesafe_sdk import ( AsyncTypeSafeClient, Choice, ChoiceAnswer, Noul, NoulAnswer, RetryPolicy, Score, ScoreAnswer, TypeSafeError, TypeSafeRateLimitError, ) from app.adk.runtime import Answer, Question logger = logging.getLogger(__name__) @dataclass(frozen=True) class JevLimits: """Every limit the client keeps. Starting values, to tune from measurement.""" max_tokens: int = 48_000 # state plus questions; the service allows 64,000 max_question_tokens: int = 24_000 # state plus the longest question; allows 32,000 request_timeout: float = 5.0 # one HTTP attempt total_timeout: float = 15.0 # a decision, retries included retries: int = 2 backoff_base: float = 0.5 backoff_cap: float = 8.0 requests_per_second: float = 5.0 burst: int = 10 max_in_flight: int = 8 breaker_failures: int = 5 breaker_window: float = 30.0 breaker_cooldown: float = 60.0 cache_size: int = 1024 budget_calls: int = 500 # per scope, usually one Execution budget_tokens: int = 1_000_000 # per scope; about $0.42 at the published price @dataclass class DecisionRecord: """One attempt to decide, kept for the evaluation report and for tuning.""" scope: str questions: tuple[str, ...] # answered | cached | too_large | over_budget | breaker_open | failed outcome: str = "failed" seconds: float = 0.0 rate_limited: bool = False input_tokens: int = 0 confidence: Mapping[str, float] = field(default_factory=dict) def estimate_tokens(text: str) -> int: """A deliberately high estimate: refusing locally is cheaper than a rejected call.""" return len(text) // 3 + 1 def encode_question(question: Question) -> Noul | Choice | Score: if question.kind == "choice": return Choice( instructions=question.instructions, criteria=dict.fromkeys(question.options), ) if question.kind == "score": return Score( instructions=question.instructions, criteria=list(question.options) ) return Noul(instructions=question.instructions) def decode_answer(question: Question, raw: object) -> Answer | None: """The SDK's answer as ours, or None when it does not answer the question asked.""" if question.kind == "choice" and isinstance(raw, ChoiceAnswer): # A choice outside the options asked for is not an answer. if raw.choice not in question.options: return None return Answer(raw.choice, raw.confidence, dict(raw.probabilities)) if question.kind == "score" and isinstance(raw, ScoreAnswer): probabilities = {str(level): p for level, p in raw.probabilities.items()} return Answer(raw.score, raw.confidence, probabilities) if question.kind == "noul" and isinstance(raw, NoulAnswer): # Jev gives a probability for a yes-or-no question and no separate # confidence, so confidence is how far the probability is from even. return Answer(raw.noul, abs(2 * raw.noul - 1)) return None class JevDecider: def __init__( self, *, api_key: str, base_url: str | None = None, model: str | None = None, limits: JevLimits | None = None, transport: Any = None, clock: Callable[[], float] = time.monotonic, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, ) -> None: self.limits = limits or JevLimits() self._clock = clock self._sleep = sleep self._client = AsyncTypeSafeClient( api_key=api_key, base_url=base_url, model=model, timeout=self.limits.request_timeout, # The SDK retries 408, 429 and 5xx, and connection and timeout errors, # with jittered backoff, and honours Retry-After. retry=RetryPolicy( max_retries=self.limits.retries, backoff_initial=self.limits.backoff_base, backoff_max=self.limits.backoff_cap, timeout=self.limits.total_timeout, ), transport=transport, ) self._in_flight = asyncio.Semaphore(self.limits.max_in_flight) # Token bucket: starts full, refills at `rate`, which drops when the service # says we are going too fast. self._rate = self.limits.requests_per_second self._tokens = float(self.limits.burst) self._refilled = clock() self._bucket = asyncio.Lock() self._failures: deque[float] = deque() self._open_until = 0.0 self._probing = False self._cache: OrderedDict[str, dict[str, Answer]] = OrderedDict() self._calls: dict[str, int] = {} self._tokens_used: dict[str, int] = {} self._over_budget: set[str] = set() self.records: list[DecisionRecord] = [] async def aclose(self) -> None: await self._client.aclose() async def decide( self, state: str, questions: Mapping[str, Question], *, scope: str = "" ) -> Mapping[str, Answer] | None: record = DecisionRecord(scope=scope, questions=tuple(questions)) self.records.append(record) started = self._clock() try: return await self._decide(state, questions, scope, record) finally: record.seconds = self._clock() - started async def _decide( self, state: str, questions: Mapping[str, Question], scope: str, record: DecisionRecord, ) -> Mapping[str, Answer] | None: if not questions: record.outcome = "answered" return {} asked = { name: { "kind": question.kind, "instructions": question.instructions, "options": list(question.options), } for name, question in questions.items() } sizes = [estimate_tokens(json.dumps(item)) for item in asked.values()] state_size = estimate_tokens(state) if ( state_size + sum(sizes) > self.limits.max_tokens or state_size + max(sizes) > self.limits.max_question_tokens ): # Never truncate: the caller must split by Deliverable or section. record.outcome = "too_large" return None key = hashlib.sha256( json.dumps({"state": state, "questions": asked}, sort_keys=True).encode() ).hexdigest() if key in self._cache: self._cache.move_to_end(key) record.outcome = "cached" return dict(self._cache[key]) if ( self._calls.get(scope, 0) >= self.limits.budget_calls or self._tokens_used.get(scope, 0) >= self.limits.budget_tokens ): if scope not in self._over_budget: self._over_budget.add(scope) logger.warning("Jev budget reached for %s; using fallbacks", scope) record.outcome = "over_budget" return None if not self._breaker_allows(): record.outcome = "breaker_open" return None self._calls[scope] = self._calls.get(scope, 0) + 1 await self._take_token() try: async with self._in_flight: response = await self._client.system_one( state=state, questions={ name: encode_question(question) for name, question in questions.items() }, ) except TypeSafeRateLimitError: # Still rate limited after the SDK's retries: halve our own rate. record.rate_limited = True self._rate = max(self._rate / 2, 0.25) self._note_failure() return None except TypeSafeError as exc: logger.info("Jev request failed: %s", type(exc).__name__) self._note_failure() return None self._note_success() used = response.usage.input_tokens or 0 record.input_tokens = used self._tokens_used[scope] = self._tokens_used.get(scope, 0) + used answers: dict[str, Answer] = {} for name, question in questions.items(): answer = decode_answer(question, response.answers.get(name)) if answer is None: # A reply that does not answer every question answers none. return None answers[name] = answer record.outcome = "answered" record.confidence = {name: a.confidence for name, a in answers.items()} self._cache[key] = answers while len(self._cache) > self.limits.cache_size: self._cache.popitem(last=False) return dict(answers) async def _take_token(self) -> None: async with self._bucket: while True: now = self._clock() self._tokens = min( float(self.limits.burst), self._tokens + (now - self._refilled) * self._rate, ) self._refilled = now if self._tokens >= 1: self._tokens -= 1 return await self._sleep((1 - self._tokens) / self._rate) def _breaker_allows(self) -> bool: now = self._clock() if now < self._open_until: return False if self._open_until and not self._probing: # Cool-down over: let exactly one call through to test the service. self._probing = True return True def _note_failure(self) -> None: now = self._clock() if self._probing: self._probing = False self._open_until = now + self.limits.breaker_cooldown return self._failures.append(now) while self._failures and now - self._failures[0] > self.limits.breaker_window: self._failures.popleft() if len(self._failures) >= self.limits.breaker_failures: self._failures.clear() self._open_until = now + self.limits.breaker_cooldown logger.warning( "Jev circuit open for %.0fs after repeated failures", self.limits.breaker_cooldown, ) def _note_success(self) -> None: self._failures.clear() self._open_until = 0.0 self._probing = False def summary(self) -> dict[str, Any]: """Totals for the evaluation report.""" outcomes: dict[str, int] = {} for record in self.records: outcomes[record.outcome] = outcomes.get(record.outcome, 0) + 1 answered = [r for r in self.records if r.outcome == "answered"] return { "decisions": len(self.records), "outcomes": outcomes, "fallbacks": sum( count for name, count in outcomes.items() if name not in {"answered", "cached"} ), "rate_limited": sum(1 for r in self.records if r.rate_limited), "seconds": round(sum(r.seconds for r in answered), 3), "input_tokens": sum(r.input_tokens for r in self.records), }