Solution architecture outcome pattern: agent and its MCP servers
Some checks failed
check agents / check (push) Failing after 13s
Some checks failed
check agents / check (push) Failing after 13s
The cp1-solution-architecture-agent and the sa-aws/azure/gcp-cloud MCP servers, a separate stream from the individual and sub-agent patterns. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,327 @@
|
||||
"""The Jev decision client: typed answers to closed questions, within strict limits.
|
||||
|
||||
Jev is TypeSafe AI's decision model. This module is the only one that speaks to it,
|
||||
through the vendor's Python SDK (`typesafe-sdk`), which owns the request shape, the
|
||||
timeout and the retries. Everything else here is ours: the service's limits are not
|
||||
published, so the client assumes they are strict and never lets Jev hold up an
|
||||
Execution. Any refusal, failure or timeout returns None and the caller takes its
|
||||
fallback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from collections import OrderedDict, deque
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from typesafe_sdk import (
|
||||
AsyncTypeSafeClient,
|
||||
Choice,
|
||||
ChoiceAnswer,
|
||||
Noul,
|
||||
NoulAnswer,
|
||||
RetryPolicy,
|
||||
Score,
|
||||
ScoreAnswer,
|
||||
TypeSafeError,
|
||||
TypeSafeRateLimitError,
|
||||
)
|
||||
|
||||
from app.engine.runtime import Answer, Question
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class JevLimits:
|
||||
"""Every limit the client keeps. Starting values, to tune from measurement."""
|
||||
|
||||
max_tokens: int = 48_000 # state plus questions; the service allows 64,000
|
||||
max_question_tokens: int = 24_000 # state plus the longest question; allows 32,000
|
||||
request_timeout: float = 5.0 # one HTTP attempt
|
||||
total_timeout: float = 15.0 # a decision, retries included
|
||||
retries: int = 2
|
||||
backoff_base: float = 0.5
|
||||
backoff_cap: float = 8.0
|
||||
requests_per_second: float = 5.0
|
||||
burst: int = 10
|
||||
max_in_flight: int = 8
|
||||
breaker_failures: int = 5
|
||||
breaker_window: float = 30.0
|
||||
breaker_cooldown: float = 60.0
|
||||
cache_size: int = 1024
|
||||
budget_calls: int = 500 # per scope, usually one Execution
|
||||
budget_tokens: int = 1_000_000 # per scope; about $0.42 at the published price
|
||||
|
||||
|
||||
@dataclass
|
||||
class DecisionRecord:
|
||||
"""One attempt to decide, kept for the evaluation report and for tuning."""
|
||||
|
||||
scope: str
|
||||
questions: tuple[str, ...]
|
||||
# answered | cached | too_large | over_budget | breaker_open | failed
|
||||
outcome: str = "failed"
|
||||
seconds: float = 0.0
|
||||
rate_limited: bool = False
|
||||
input_tokens: int = 0
|
||||
confidence: Mapping[str, float] = field(default_factory=dict)
|
||||
|
||||
|
||||
def estimate_tokens(text: str) -> int:
|
||||
"""A deliberately high estimate: refusing locally is cheaper than a rejected call."""
|
||||
return len(text) // 3 + 1
|
||||
|
||||
|
||||
def encode_question(question: Question) -> Noul | Choice | Score:
|
||||
if question.kind == "choice":
|
||||
return Choice(
|
||||
instructions=question.instructions,
|
||||
criteria=dict.fromkeys(question.options),
|
||||
)
|
||||
if question.kind == "score":
|
||||
return Score(
|
||||
instructions=question.instructions, criteria=list(question.options)
|
||||
)
|
||||
return Noul(instructions=question.instructions)
|
||||
|
||||
|
||||
def decode_answer(question: Question, raw: object) -> Answer | None:
|
||||
"""The SDK's answer as ours, or None when it does not answer the question asked."""
|
||||
if question.kind == "choice" and isinstance(raw, ChoiceAnswer):
|
||||
# A choice outside the options asked for is not an answer.
|
||||
if raw.choice not in question.options:
|
||||
return None
|
||||
return Answer(raw.choice, raw.confidence, dict(raw.probabilities))
|
||||
if question.kind == "score" and isinstance(raw, ScoreAnswer):
|
||||
probabilities = {str(level): p for level, p in raw.probabilities.items()}
|
||||
return Answer(raw.score, raw.confidence, probabilities)
|
||||
if question.kind == "noul" and isinstance(raw, NoulAnswer):
|
||||
# Jev gives a probability for a yes-or-no question and no separate
|
||||
# confidence, so confidence is how far the probability is from even.
|
||||
return Answer(raw.noul, abs(2 * raw.noul - 1))
|
||||
return None
|
||||
|
||||
|
||||
class JevDecider:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
api_key: str,
|
||||
base_url: str | None = None,
|
||||
model: str | None = None,
|
||||
limits: JevLimits | None = None,
|
||||
transport: Any = None,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
) -> None:
|
||||
self.limits = limits or JevLimits()
|
||||
self._clock = clock
|
||||
self._sleep = sleep
|
||||
self._client = AsyncTypeSafeClient(
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
model=model,
|
||||
timeout=self.limits.request_timeout,
|
||||
# The SDK retries 408, 429 and 5xx, and connection and timeout errors,
|
||||
# with jittered backoff, and honours Retry-After.
|
||||
retry=RetryPolicy(
|
||||
max_retries=self.limits.retries,
|
||||
backoff_initial=self.limits.backoff_base,
|
||||
backoff_max=self.limits.backoff_cap,
|
||||
timeout=self.limits.total_timeout,
|
||||
),
|
||||
transport=transport,
|
||||
)
|
||||
self._in_flight = asyncio.Semaphore(self.limits.max_in_flight)
|
||||
# Token bucket: starts full, refills at `rate`, which drops when the service
|
||||
# says we are going too fast.
|
||||
self._rate = self.limits.requests_per_second
|
||||
self._tokens = float(self.limits.burst)
|
||||
self._refilled = clock()
|
||||
self._bucket = asyncio.Lock()
|
||||
self._failures: deque[float] = deque()
|
||||
self._open_until = 0.0
|
||||
self._probing = False
|
||||
self._cache: OrderedDict[str, dict[str, Answer]] = OrderedDict()
|
||||
self._calls: dict[str, int] = {}
|
||||
self._tokens_used: dict[str, int] = {}
|
||||
self._over_budget: set[str] = set()
|
||||
self.records: list[DecisionRecord] = []
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._client.aclose()
|
||||
|
||||
async def decide(
|
||||
self, state: str, questions: Mapping[str, Question], *, scope: str = ""
|
||||
) -> Mapping[str, Answer] | None:
|
||||
record = DecisionRecord(scope=scope, questions=tuple(questions))
|
||||
self.records.append(record)
|
||||
started = self._clock()
|
||||
try:
|
||||
return await self._decide(state, questions, scope, record)
|
||||
finally:
|
||||
record.seconds = self._clock() - started
|
||||
|
||||
async def _decide(
|
||||
self,
|
||||
state: str,
|
||||
questions: Mapping[str, Question],
|
||||
scope: str,
|
||||
record: DecisionRecord,
|
||||
) -> Mapping[str, Answer] | None:
|
||||
if not questions:
|
||||
record.outcome = "answered"
|
||||
return {}
|
||||
|
||||
asked = {
|
||||
name: {
|
||||
"kind": question.kind,
|
||||
"instructions": question.instructions,
|
||||
"options": list(question.options),
|
||||
}
|
||||
for name, question in questions.items()
|
||||
}
|
||||
sizes = [estimate_tokens(json.dumps(item)) for item in asked.values()]
|
||||
state_size = estimate_tokens(state)
|
||||
if (
|
||||
state_size + sum(sizes) > self.limits.max_tokens
|
||||
or state_size + max(sizes) > self.limits.max_question_tokens
|
||||
):
|
||||
# Never truncate: the caller must split by Deliverable or section.
|
||||
record.outcome = "too_large"
|
||||
return None
|
||||
|
||||
key = hashlib.sha256(
|
||||
json.dumps({"state": state, "questions": asked}, sort_keys=True).encode()
|
||||
).hexdigest()
|
||||
if key in self._cache:
|
||||
self._cache.move_to_end(key)
|
||||
record.outcome = "cached"
|
||||
return dict(self._cache[key])
|
||||
|
||||
if (
|
||||
self._calls.get(scope, 0) >= self.limits.budget_calls
|
||||
or self._tokens_used.get(scope, 0) >= self.limits.budget_tokens
|
||||
):
|
||||
if scope not in self._over_budget:
|
||||
self._over_budget.add(scope)
|
||||
logger.warning("Jev budget reached for %s; using fallbacks", scope)
|
||||
record.outcome = "over_budget"
|
||||
return None
|
||||
|
||||
if not self._breaker_allows():
|
||||
record.outcome = "breaker_open"
|
||||
return None
|
||||
|
||||
self._calls[scope] = self._calls.get(scope, 0) + 1
|
||||
await self._take_token()
|
||||
try:
|
||||
async with self._in_flight:
|
||||
response = await self._client.system_one(
|
||||
state=state,
|
||||
questions={
|
||||
name: encode_question(question)
|
||||
for name, question in questions.items()
|
||||
},
|
||||
)
|
||||
except TypeSafeRateLimitError:
|
||||
# Still rate limited after the SDK's retries: halve our own rate.
|
||||
record.rate_limited = True
|
||||
self._rate = max(self._rate / 2, 0.25)
|
||||
self._note_failure()
|
||||
return None
|
||||
except TypeSafeError as exc:
|
||||
logger.info("Jev request failed: %s", type(exc).__name__)
|
||||
self._note_failure()
|
||||
return None
|
||||
self._note_success()
|
||||
|
||||
used = response.usage.input_tokens or 0
|
||||
record.input_tokens = used
|
||||
self._tokens_used[scope] = self._tokens_used.get(scope, 0) + used
|
||||
|
||||
answers: dict[str, Answer] = {}
|
||||
for name, question in questions.items():
|
||||
answer = decode_answer(question, response.answers.get(name))
|
||||
if answer is None:
|
||||
# A reply that does not answer every question answers none.
|
||||
return None
|
||||
answers[name] = answer
|
||||
|
||||
record.outcome = "answered"
|
||||
record.confidence = {name: a.confidence for name, a in answers.items()}
|
||||
self._cache[key] = answers
|
||||
while len(self._cache) > self.limits.cache_size:
|
||||
self._cache.popitem(last=False)
|
||||
return dict(answers)
|
||||
|
||||
async def _take_token(self) -> None:
|
||||
async with self._bucket:
|
||||
while True:
|
||||
now = self._clock()
|
||||
self._tokens = min(
|
||||
float(self.limits.burst),
|
||||
self._tokens + (now - self._refilled) * self._rate,
|
||||
)
|
||||
self._refilled = now
|
||||
if self._tokens >= 1:
|
||||
self._tokens -= 1
|
||||
return
|
||||
await self._sleep((1 - self._tokens) / self._rate)
|
||||
|
||||
def _breaker_allows(self) -> bool:
|
||||
now = self._clock()
|
||||
if now < self._open_until:
|
||||
return False
|
||||
if self._open_until and not self._probing:
|
||||
# Cool-down over: let exactly one call through to test the service.
|
||||
self._probing = True
|
||||
return True
|
||||
|
||||
def _note_failure(self) -> None:
|
||||
now = self._clock()
|
||||
if self._probing:
|
||||
self._probing = False
|
||||
self._open_until = now + self.limits.breaker_cooldown
|
||||
return
|
||||
self._failures.append(now)
|
||||
while self._failures and now - self._failures[0] > self.limits.breaker_window:
|
||||
self._failures.popleft()
|
||||
if len(self._failures) >= self.limits.breaker_failures:
|
||||
self._failures.clear()
|
||||
self._open_until = now + self.limits.breaker_cooldown
|
||||
logger.warning(
|
||||
"Jev circuit open for %.0fs after repeated failures",
|
||||
self.limits.breaker_cooldown,
|
||||
)
|
||||
|
||||
def _note_success(self) -> None:
|
||||
self._failures.clear()
|
||||
self._open_until = 0.0
|
||||
self._probing = False
|
||||
|
||||
def summary(self) -> dict[str, Any]:
|
||||
"""Totals for the evaluation report."""
|
||||
outcomes: dict[str, int] = {}
|
||||
for record in self.records:
|
||||
outcomes[record.outcome] = outcomes.get(record.outcome, 0) + 1
|
||||
answered = [r for r in self.records if r.outcome == "answered"]
|
||||
return {
|
||||
"decisions": len(self.records),
|
||||
"outcomes": outcomes,
|
||||
"fallbacks": sum(
|
||||
count
|
||||
for name, count in outcomes.items()
|
||||
if name not in {"answered", "cached"}
|
||||
),
|
||||
"rate_limited": sum(1 for r in self.records if r.rate_limited),
|
||||
"seconds": round(sum(r.seconds for r in answered), 3),
|
||||
"input_tokens": sum(r.input_tokens for r in self.records),
|
||||
}
|
||||
Reference in New Issue
Block a user