Some checks failed
validation / verify (push) Failing after 11s
A pillar's checklist item the request gives no information about was marked "not addressed", which reads as a fault in the design. It is missing information. The checklist review gains a fourth outcome, to confirm with the customer, which must carry the question to ask. The questions are written into the design recommendations and the guide, and to customer-questions.md; the evaluation's judged check counts such an item as accounted for. Records the owner's decisions of 2026-10-10 and the vendor's answers on Jev's retention, training, context and rate limits (iteration 5 step 1), and the owner's labels, which showed the judged question was wrong, not the judges. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
82 lines
3.1 KiB
Python
82 lines
3.1 KiB
Python
"""Every Execution judged as it ends (ADR-0007, iteration 5 step 19).
|
|
|
|
The evaluation scores benchmark cases. This asks the same graded questions of a live
|
|
Execution's design recommendations when it ends, and keeps the answers with its
|
|
record. It reports; it never stops or changes an Execution.
|
|
|
|
Without a decision an Execution carries no assessment.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import Mapping
|
|
from typing import Any
|
|
|
|
from app.adk.runtime import Decider, Question
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
GRADES = ("not at all", "weakly", "mostly", "fully")
|
|
QUALITY_BAR = 0.67 # mean of the graded rubric: "mostly" or better
|
|
QUALITY_RUBRIC: Mapping[str, str] = {
|
|
"specific": "Are the recommendations specific to this workload and its stated "
|
|
"requirements, as opposed to generic best practice that would fit any workload?",
|
|
"actionable": "Does each recommendation name the concrete setting, feature or "
|
|
"configuration to apply, so an engineer could act on it without more research?",
|
|
"trade_offs": "Where pillars pull in different directions (for example cost "
|
|
"against reliability), is the trade-off named and a resolution recommended?",
|
|
"traceable": "Can each recommendation be traced to a requirement or constraint "
|
|
"of the workload?",
|
|
}
|
|
_JUDGED = "design_recommendations"
|
|
|
|
|
|
def checklist_question(pillar: str, item: str) -> str:
|
|
"""Whether the recommendations account for one item of a pillar's checklist.
|
|
|
|
An item is accounted for in three ways. The third matters: a customer's request
|
|
seldom says enough to settle every item, and what it leaves unsaid is a
|
|
question for them, not a fault in the design (the owner, 2026-10-10).
|
|
"""
|
|
return (
|
|
f"Do the recommendations account for this {pillar} checklist item in any "
|
|
"of these ways: by recommending it, by explaining why it does not apply, "
|
|
"or by recording it as something to confirm with the customer? "
|
|
f"{item}"
|
|
)
|
|
|
|
|
|
async def assess(
|
|
decider: Decider,
|
|
deliverables: Mapping[str, Mapping[str, Any]],
|
|
*,
|
|
scope: str = "",
|
|
) -> dict[str, Any] | None:
|
|
"""The rubric's scores for the design recommendations, each from 0 to 1, with
|
|
their mean and whether it meets the bar. None when there is nothing to judge or
|
|
no decision."""
|
|
document = str((deliverables.get(_JUDGED) or {}).get("content", ""))
|
|
if not document.strip():
|
|
return None
|
|
answers = await decider.decide(
|
|
document,
|
|
{key: Question("score", text, GRADES) for key, text in QUALITY_RUBRIC.items()},
|
|
scope=scope,
|
|
)
|
|
if answers is None:
|
|
return None
|
|
top = len(GRADES) - 1
|
|
scores = {
|
|
key: round(min(1.0, max(0.0, float(answers[key].value) / top)), 2)
|
|
for key in QUALITY_RUBRIC
|
|
}
|
|
mean = round(sum(scores.values()) / len(scores), 2)
|
|
if mean < QUALITY_BAR:
|
|
logger.warning("[%s] design recommendations judged at %.2f", scope, mean)
|
|
return {
|
|
"recommendations_quality": scores,
|
|
"mean": mean,
|
|
"meets_bar": mean >= QUALITY_BAR,
|
|
}
|