328 lines
12 KiB
Python
328 lines
12 KiB
Python
"""The Jev decision client: typed answers to closed questions, within strict limits.
|
|
|
|
Jev is TypeSafe AI's decision model. This module is the only one that speaks to it,
|
|
through the vendor's Python SDK (`typesafe-sdk`), which owns the request shape, the
|
|
timeout and the retries. Everything else here is ours: the service's limits are not
|
|
published, so the client assumes they are strict and never lets Jev hold up an
|
|
Execution. Any refusal, failure or timeout returns None and the caller takes its
|
|
fallback.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import time
|
|
from collections import OrderedDict, deque
|
|
from collections.abc import Awaitable, Callable, Mapping
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from typesafe_sdk import (
|
|
AsyncTypeSafeClient,
|
|
Choice,
|
|
ChoiceAnswer,
|
|
Noul,
|
|
NoulAnswer,
|
|
RetryPolicy,
|
|
Score,
|
|
ScoreAnswer,
|
|
TypeSafeError,
|
|
TypeSafeRateLimitError,
|
|
)
|
|
|
|
from execution_engine.adk.runtime import Answer, Question
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class JevLimits:
|
|
"""Every limit the client keeps. Starting values, to tune from measurement."""
|
|
|
|
max_tokens: int = 48_000 # state plus questions; the service allows 64,000
|
|
max_question_tokens: int = 24_000 # state plus the longest question; allows 32,000
|
|
request_timeout: float = 5.0 # one HTTP attempt
|
|
total_timeout: float = 15.0 # a decision, retries included
|
|
retries: int = 2
|
|
backoff_base: float = 0.5
|
|
backoff_cap: float = 8.0
|
|
requests_per_second: float = 5.0
|
|
burst: int = 10
|
|
max_in_flight: int = 8
|
|
breaker_failures: int = 5
|
|
breaker_window: float = 30.0
|
|
breaker_cooldown: float = 60.0
|
|
cache_size: int = 1024
|
|
budget_calls: int = 500 # per scope, usually one Execution
|
|
budget_tokens: int = 1_000_000 # per scope; about $0.42 at the published price
|
|
|
|
|
|
@dataclass
|
|
class DecisionRecord:
|
|
"""One attempt to decide, kept for the evaluation report and for tuning."""
|
|
|
|
scope: str
|
|
questions: tuple[str, ...]
|
|
# answered | cached | too_large | over_budget | breaker_open | failed
|
|
outcome: str = "failed"
|
|
seconds: float = 0.0
|
|
rate_limited: bool = False
|
|
input_tokens: int = 0
|
|
confidence: Mapping[str, float] = field(default_factory=dict)
|
|
|
|
|
|
def estimate_tokens(text: str) -> int:
|
|
"""A deliberately high estimate: refusing locally is cheaper than a rejected call."""
|
|
return len(text) // 3 + 1
|
|
|
|
|
|
def encode_question(question: Question) -> Noul | Choice | Score:
|
|
if question.kind == "choice":
|
|
return Choice(
|
|
instructions=question.instructions,
|
|
criteria=dict.fromkeys(question.options),
|
|
)
|
|
if question.kind == "score":
|
|
return Score(
|
|
instructions=question.instructions, criteria=list(question.options)
|
|
)
|
|
return Noul(instructions=question.instructions)
|
|
|
|
|
|
def decode_answer(question: Question, raw: object) -> Answer | None:
|
|
"""The SDK's answer as ours, or None when it does not answer the question asked."""
|
|
if question.kind == "choice" and isinstance(raw, ChoiceAnswer):
|
|
# A choice outside the options asked for is not an answer.
|
|
if raw.choice not in question.options:
|
|
return None
|
|
return Answer(raw.choice, raw.confidence, dict(raw.probabilities))
|
|
if question.kind == "score" and isinstance(raw, ScoreAnswer):
|
|
probabilities = {str(level): p for level, p in raw.probabilities.items()}
|
|
return Answer(raw.score, raw.confidence, probabilities)
|
|
if question.kind == "noul" and isinstance(raw, NoulAnswer):
|
|
# Jev gives a probability for a yes-or-no question and no separate
|
|
# confidence, so confidence is how far the probability is from even.
|
|
return Answer(raw.noul, abs(2 * raw.noul - 1))
|
|
return None
|
|
|
|
|
|
class JevDecider:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
api_key: str,
|
|
base_url: str | None = None,
|
|
model: str | None = None,
|
|
limits: JevLimits | None = None,
|
|
transport: Any = None,
|
|
clock: Callable[[], float] = time.monotonic,
|
|
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
|
) -> None:
|
|
self.limits = limits or JevLimits()
|
|
self._clock = clock
|
|
self._sleep = sleep
|
|
self._client = AsyncTypeSafeClient(
|
|
api_key=api_key,
|
|
base_url=base_url,
|
|
model=model,
|
|
timeout=self.limits.request_timeout,
|
|
# The SDK retries 408, 429 and 5xx, and connection and timeout errors,
|
|
# with jittered backoff, and honours Retry-After.
|
|
retry=RetryPolicy(
|
|
max_retries=self.limits.retries,
|
|
backoff_initial=self.limits.backoff_base,
|
|
backoff_max=self.limits.backoff_cap,
|
|
timeout=self.limits.total_timeout,
|
|
),
|
|
transport=transport,
|
|
)
|
|
self._in_flight = asyncio.Semaphore(self.limits.max_in_flight)
|
|
# Token bucket: starts full, refills at `rate`, which drops when the service
|
|
# says we are going too fast.
|
|
self._rate = self.limits.requests_per_second
|
|
self._tokens = float(self.limits.burst)
|
|
self._refilled = clock()
|
|
self._bucket = asyncio.Lock()
|
|
self._failures: deque[float] = deque()
|
|
self._open_until = 0.0
|
|
self._probing = False
|
|
self._cache: OrderedDict[str, dict[str, Answer]] = OrderedDict()
|
|
self._calls: dict[str, int] = {}
|
|
self._tokens_used: dict[str, int] = {}
|
|
self._over_budget: set[str] = set()
|
|
self.records: list[DecisionRecord] = []
|
|
|
|
async def aclose(self) -> None:
|
|
await self._client.aclose()
|
|
|
|
async def decide(
|
|
self, state: str, questions: Mapping[str, Question], *, scope: str = ""
|
|
) -> Mapping[str, Answer] | None:
|
|
record = DecisionRecord(scope=scope, questions=tuple(questions))
|
|
self.records.append(record)
|
|
started = self._clock()
|
|
try:
|
|
return await self._decide(state, questions, scope, record)
|
|
finally:
|
|
record.seconds = self._clock() - started
|
|
|
|
async def _decide(
|
|
self,
|
|
state: str,
|
|
questions: Mapping[str, Question],
|
|
scope: str,
|
|
record: DecisionRecord,
|
|
) -> Mapping[str, Answer] | None:
|
|
if not questions:
|
|
record.outcome = "answered"
|
|
return {}
|
|
|
|
asked = {
|
|
name: {
|
|
"kind": question.kind,
|
|
"instructions": question.instructions,
|
|
"options": list(question.options),
|
|
}
|
|
for name, question in questions.items()
|
|
}
|
|
sizes = [estimate_tokens(json.dumps(item)) for item in asked.values()]
|
|
state_size = estimate_tokens(state)
|
|
if (
|
|
state_size + sum(sizes) > self.limits.max_tokens
|
|
or state_size + max(sizes) > self.limits.max_question_tokens
|
|
):
|
|
# Never truncate: the caller must split by Deliverable or section.
|
|
record.outcome = "too_large"
|
|
return None
|
|
|
|
key = hashlib.sha256(
|
|
json.dumps({"state": state, "questions": asked}, sort_keys=True).encode()
|
|
).hexdigest()
|
|
if key in self._cache:
|
|
self._cache.move_to_end(key)
|
|
record.outcome = "cached"
|
|
return dict(self._cache[key])
|
|
|
|
if (
|
|
self._calls.get(scope, 0) >= self.limits.budget_calls
|
|
or self._tokens_used.get(scope, 0) >= self.limits.budget_tokens
|
|
):
|
|
if scope not in self._over_budget:
|
|
self._over_budget.add(scope)
|
|
logger.warning("Jev budget reached for %s; using fallbacks", scope)
|
|
record.outcome = "over_budget"
|
|
return None
|
|
|
|
if not self._breaker_allows():
|
|
record.outcome = "breaker_open"
|
|
return None
|
|
|
|
self._calls[scope] = self._calls.get(scope, 0) + 1
|
|
await self._take_token()
|
|
try:
|
|
async with self._in_flight:
|
|
response = await self._client.system_one(
|
|
state=state,
|
|
questions={
|
|
name: encode_question(question)
|
|
for name, question in questions.items()
|
|
},
|
|
)
|
|
except TypeSafeRateLimitError:
|
|
# Still rate limited after the SDK's retries: halve our own rate.
|
|
record.rate_limited = True
|
|
self._rate = max(self._rate / 2, 0.25)
|
|
self._note_failure()
|
|
return None
|
|
except TypeSafeError as exc:
|
|
logger.info("Jev request failed: %s", type(exc).__name__)
|
|
self._note_failure()
|
|
return None
|
|
self._note_success()
|
|
|
|
used = response.usage.input_tokens or 0
|
|
record.input_tokens = used
|
|
self._tokens_used[scope] = self._tokens_used.get(scope, 0) + used
|
|
|
|
answers: dict[str, Answer] = {}
|
|
for name, question in questions.items():
|
|
answer = decode_answer(question, response.answers.get(name))
|
|
if answer is None:
|
|
# A reply that does not answer every question answers none.
|
|
return None
|
|
answers[name] = answer
|
|
|
|
record.outcome = "answered"
|
|
record.confidence = {name: a.confidence for name, a in answers.items()}
|
|
self._cache[key] = answers
|
|
while len(self._cache) > self.limits.cache_size:
|
|
self._cache.popitem(last=False)
|
|
return dict(answers)
|
|
|
|
async def _take_token(self) -> None:
|
|
async with self._bucket:
|
|
while True:
|
|
now = self._clock()
|
|
self._tokens = min(
|
|
float(self.limits.burst),
|
|
self._tokens + (now - self._refilled) * self._rate,
|
|
)
|
|
self._refilled = now
|
|
if self._tokens >= 1:
|
|
self._tokens -= 1
|
|
return
|
|
await self._sleep((1 - self._tokens) / self._rate)
|
|
|
|
def _breaker_allows(self) -> bool:
|
|
now = self._clock()
|
|
if now < self._open_until:
|
|
return False
|
|
if self._open_until and not self._probing:
|
|
# Cool-down over: let exactly one call through to test the service.
|
|
self._probing = True
|
|
return True
|
|
|
|
def _note_failure(self) -> None:
|
|
now = self._clock()
|
|
if self._probing:
|
|
self._probing = False
|
|
self._open_until = now + self.limits.breaker_cooldown
|
|
return
|
|
self._failures.append(now)
|
|
while self._failures and now - self._failures[0] > self.limits.breaker_window:
|
|
self._failures.popleft()
|
|
if len(self._failures) >= self.limits.breaker_failures:
|
|
self._failures.clear()
|
|
self._open_until = now + self.limits.breaker_cooldown
|
|
logger.warning(
|
|
"Jev circuit open for %.0fs after repeated failures",
|
|
self.limits.breaker_cooldown,
|
|
)
|
|
|
|
def _note_success(self) -> None:
|
|
self._failures.clear()
|
|
self._open_until = 0.0
|
|
self._probing = False
|
|
|
|
def summary(self) -> dict[str, Any]:
|
|
"""Totals for the evaluation report."""
|
|
outcomes: dict[str, int] = {}
|
|
for record in self.records:
|
|
outcomes[record.outcome] = outcomes.get(record.outcome, 0) + 1
|
|
answered = [r for r in self.records if r.outcome == "answered"]
|
|
return {
|
|
"decisions": len(self.records),
|
|
"outcomes": outcomes,
|
|
"fallbacks": sum(
|
|
count
|
|
for name, count in outcomes.items()
|
|
if name not in {"answered", "cached"}
|
|
),
|
|
"rate_limited": sum(1 for r in self.records if r.rate_limited),
|
|
"seconds": round(sum(r.seconds for r in answered), 3),
|
|
"input_tokens": sum(r.input_tokens for r in self.records),
|
|
}
|