Files
orchestration_engine/execution_engine/tools/jev.py

328 lines
12 KiB
Python

"""The Jev decision client: typed answers to closed questions, within strict limits.
Jev is TypeSafe AI's decision model. This module is the only one that speaks to it,
through the vendor's Python SDK (`typesafe-sdk`), which owns the request shape, the
timeout and the retries. Everything else here is ours: the service's limits are not
published, so the client assumes they are strict and never lets Jev hold up an
Execution. Any refusal, failure or timeout returns None and the caller takes its
fallback.
"""
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import time
from collections import OrderedDict, deque
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass, field
from typing import Any
from typesafe_sdk import (
AsyncTypeSafeClient,
Choice,
ChoiceAnswer,
Noul,
NoulAnswer,
RetryPolicy,
Score,
ScoreAnswer,
TypeSafeError,
TypeSafeRateLimitError,
)
from execution_engine.adk.runtime import Answer, Question
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class JevLimits:
"""Every limit the client keeps. Starting values, to tune from measurement."""
max_tokens: int = 48_000 # state plus questions; the service allows 64,000
max_question_tokens: int = 24_000 # state plus the longest question; allows 32,000
request_timeout: float = 5.0 # one HTTP attempt
total_timeout: float = 15.0 # a decision, retries included
retries: int = 2
backoff_base: float = 0.5
backoff_cap: float = 8.0
requests_per_second: float = 5.0
burst: int = 10
max_in_flight: int = 8
breaker_failures: int = 5
breaker_window: float = 30.0
breaker_cooldown: float = 60.0
cache_size: int = 1024
budget_calls: int = 500 # per scope, usually one Execution
budget_tokens: int = 1_000_000 # per scope; about $0.42 at the published price
@dataclass
class DecisionRecord:
"""One attempt to decide, kept for the evaluation report and for tuning."""
scope: str
questions: tuple[str, ...]
# answered | cached | too_large | over_budget | breaker_open | failed
outcome: str = "failed"
seconds: float = 0.0
rate_limited: bool = False
input_tokens: int = 0
confidence: Mapping[str, float] = field(default_factory=dict)
def estimate_tokens(text: str) -> int:
"""A deliberately high estimate: refusing locally is cheaper than a rejected call."""
return len(text) // 3 + 1
def encode_question(question: Question) -> Noul | Choice | Score:
if question.kind == "choice":
return Choice(
instructions=question.instructions,
criteria=dict.fromkeys(question.options),
)
if question.kind == "score":
return Score(
instructions=question.instructions, criteria=list(question.options)
)
return Noul(instructions=question.instructions)
def decode_answer(question: Question, raw: object) -> Answer | None:
"""The SDK's answer as ours, or None when it does not answer the question asked."""
if question.kind == "choice" and isinstance(raw, ChoiceAnswer):
# A choice outside the options asked for is not an answer.
if raw.choice not in question.options:
return None
return Answer(raw.choice, raw.confidence, dict(raw.probabilities))
if question.kind == "score" and isinstance(raw, ScoreAnswer):
probabilities = {str(level): p for level, p in raw.probabilities.items()}
return Answer(raw.score, raw.confidence, probabilities)
if question.kind == "noul" and isinstance(raw, NoulAnswer):
# Jev gives a probability for a yes-or-no question and no separate
# confidence, so confidence is how far the probability is from even.
return Answer(raw.noul, abs(2 * raw.noul - 1))
return None
class JevDecider:
def __init__(
self,
*,
api_key: str,
base_url: str | None = None,
model: str | None = None,
limits: JevLimits | None = None,
transport: Any = None,
clock: Callable[[], float] = time.monotonic,
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
) -> None:
self.limits = limits or JevLimits()
self._clock = clock
self._sleep = sleep
self._client = AsyncTypeSafeClient(
api_key=api_key,
base_url=base_url,
model=model,
timeout=self.limits.request_timeout,
# The SDK retries 408, 429 and 5xx, and connection and timeout errors,
# with jittered backoff, and honours Retry-After.
retry=RetryPolicy(
max_retries=self.limits.retries,
backoff_initial=self.limits.backoff_base,
backoff_max=self.limits.backoff_cap,
timeout=self.limits.total_timeout,
),
transport=transport,
)
self._in_flight = asyncio.Semaphore(self.limits.max_in_flight)
# Token bucket: starts full, refills at `rate`, which drops when the service
# says we are going too fast.
self._rate = self.limits.requests_per_second
self._tokens = float(self.limits.burst)
self._refilled = clock()
self._bucket = asyncio.Lock()
self._failures: deque[float] = deque()
self._open_until = 0.0
self._probing = False
self._cache: OrderedDict[str, dict[str, Answer]] = OrderedDict()
self._calls: dict[str, int] = {}
self._tokens_used: dict[str, int] = {}
self._over_budget: set[str] = set()
self.records: list[DecisionRecord] = []
async def aclose(self) -> None:
await self._client.aclose()
async def decide(
self, state: str, questions: Mapping[str, Question], *, scope: str = ""
) -> Mapping[str, Answer] | None:
record = DecisionRecord(scope=scope, questions=tuple(questions))
self.records.append(record)
started = self._clock()
try:
return await self._decide(state, questions, scope, record)
finally:
record.seconds = self._clock() - started
async def _decide(
self,
state: str,
questions: Mapping[str, Question],
scope: str,
record: DecisionRecord,
) -> Mapping[str, Answer] | None:
if not questions:
record.outcome = "answered"
return {}
asked = {
name: {
"kind": question.kind,
"instructions": question.instructions,
"options": list(question.options),
}
for name, question in questions.items()
}
sizes = [estimate_tokens(json.dumps(item)) for item in asked.values()]
state_size = estimate_tokens(state)
if (
state_size + sum(sizes) > self.limits.max_tokens
or state_size + max(sizes) > self.limits.max_question_tokens
):
# Never truncate: the caller must split by Deliverable or section.
record.outcome = "too_large"
return None
key = hashlib.sha256(
json.dumps({"state": state, "questions": asked}, sort_keys=True).encode()
).hexdigest()
if key in self._cache:
self._cache.move_to_end(key)
record.outcome = "cached"
return dict(self._cache[key])
if (
self._calls.get(scope, 0) >= self.limits.budget_calls
or self._tokens_used.get(scope, 0) >= self.limits.budget_tokens
):
if scope not in self._over_budget:
self._over_budget.add(scope)
logger.warning("Jev budget reached for %s; using fallbacks", scope)
record.outcome = "over_budget"
return None
if not self._breaker_allows():
record.outcome = "breaker_open"
return None
self._calls[scope] = self._calls.get(scope, 0) + 1
await self._take_token()
try:
async with self._in_flight:
response = await self._client.system_one(
state=state,
questions={
name: encode_question(question)
for name, question in questions.items()
},
)
except TypeSafeRateLimitError:
# Still rate limited after the SDK's retries: halve our own rate.
record.rate_limited = True
self._rate = max(self._rate / 2, 0.25)
self._note_failure()
return None
except TypeSafeError as exc:
logger.info("Jev request failed: %s", type(exc).__name__)
self._note_failure()
return None
self._note_success()
used = response.usage.input_tokens or 0
record.input_tokens = used
self._tokens_used[scope] = self._tokens_used.get(scope, 0) + used
answers: dict[str, Answer] = {}
for name, question in questions.items():
answer = decode_answer(question, response.answers.get(name))
if answer is None:
# A reply that does not answer every question answers none.
return None
answers[name] = answer
record.outcome = "answered"
record.confidence = {name: a.confidence for name, a in answers.items()}
self._cache[key] = answers
while len(self._cache) > self.limits.cache_size:
self._cache.popitem(last=False)
return dict(answers)
async def _take_token(self) -> None:
async with self._bucket:
while True:
now = self._clock()
self._tokens = min(
float(self.limits.burst),
self._tokens + (now - self._refilled) * self._rate,
)
self._refilled = now
if self._tokens >= 1:
self._tokens -= 1
return
await self._sleep((1 - self._tokens) / self._rate)
def _breaker_allows(self) -> bool:
now = self._clock()
if now < self._open_until:
return False
if self._open_until and not self._probing:
# Cool-down over: let exactly one call through to test the service.
self._probing = True
return True
def _note_failure(self) -> None:
now = self._clock()
if self._probing:
self._probing = False
self._open_until = now + self.limits.breaker_cooldown
return
self._failures.append(now)
while self._failures and now - self._failures[0] > self.limits.breaker_window:
self._failures.popleft()
if len(self._failures) >= self.limits.breaker_failures:
self._failures.clear()
self._open_until = now + self.limits.breaker_cooldown
logger.warning(
"Jev circuit open for %.0fs after repeated failures",
self.limits.breaker_cooldown,
)
def _note_success(self) -> None:
self._failures.clear()
self._open_until = 0.0
self._probing = False
def summary(self) -> dict[str, Any]:
"""Totals for the evaluation report."""
outcomes: dict[str, int] = {}
for record in self.records:
outcomes[record.outcome] = outcomes.get(record.outcome, 0) + 1
answered = [r for r in self.records if r.outcome == "answered"]
return {
"decisions": len(self.records),
"outcomes": outcomes,
"fallbacks": sum(
count
for name, count in outcomes.items()
if name not in {"answered", "cached"}
),
"rate_limited": sum(1 for r in self.records if r.rate_limited),
"seconds": round(sum(r.seconds for r in answered), 3),
"input_tokens": sum(r.input_tokens for r in self.records),
}