feat: establish the Target architecture as a graph before code
Design is now approved before any infrastructure code is written (ADR-0005). A new target-architect Specialist produces one graph of Elements, Relationships and Checks; the diagram is rendered from it, deployment guidance must account for every Check, and validation gives a verdict per Check. Requirements, components, products and recommendations carry ids in data files, and a submission with a gap in the Requirements chain is refused. Also in this commit: - Customer cases group Executions, with an intent history per case - lesson candidates are captured when an Execution ends - evaluation gains chain, verdict, judged and interactive checks, and per-case thresholds - Terraform providers are cached between checks - the requirements analyst proposes defaults instead of asking - the legacy-baseline case and the checks that only read it are removed - ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed - plans for iterations 4 to 7 under .agents/plans/ Unproven with a real model: no evaluation run has completed on this design. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
77
eval/judge.py
Normal file
77
eval/judge.py
Normal file
@@ -0,0 +1,77 @@
|
||||
"""Judges: graded answers about a Deliverable that string matching cannot give.
|
||||
|
||||
A judge takes a document and a set of questions and returns a score from 0 to 1 per
|
||||
question. The shape (one state, many typed questions) is deliberately the one a
|
||||
decision model takes, so the Gemini judge here can be swapped for one later.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Protocol
|
||||
|
||||
INSTRUCTIONS = (
|
||||
"You grade a document from a cloud solution architecture engagement. Answer every "
|
||||
"question about the document only, strictly and without benefit of the doubt. "
|
||||
'Reply with one JSON object mapping each question id to a number: for a "yes_no" '
|
||||
'question, 1 for yes and 0 for no; for a "graded" question, 0 (not at all), 0.33 '
|
||||
"(weakly), 0.67 (mostly) or 1 (fully)."
|
||||
)
|
||||
|
||||
|
||||
class Judge(Protocol):
|
||||
async def answer(
|
||||
self, document: str, questions: Mapping[str, Mapping[str, str]]
|
||||
) -> Mapping[str, float]:
|
||||
"""Score each question, keyed by id, from 0 to 1.
|
||||
|
||||
`questions` maps an id to {"kind": "yes_no" | "graded", "text": ...}.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class GeminiJudge:
|
||||
"""A judge on the Gemini API or Vertex AI, chosen by the environment as ADK does."""
|
||||
|
||||
def __init__(self, model: str) -> None:
|
||||
from google import genai
|
||||
|
||||
self._client = genai.Client()
|
||||
self._model = model
|
||||
|
||||
async def answer(
|
||||
self, document: str, questions: Mapping[str, Mapping[str, str]]
|
||||
) -> Mapping[str, float]:
|
||||
from google.genai import types
|
||||
|
||||
prompt = (
|
||||
f"{INSTRUCTIONS}\n\nQuestions:\n"
|
||||
f"{json.dumps(dict(questions), indent=2, ensure_ascii=False)}\n\n"
|
||||
f"Document:\n<document>\n{document}\n</document>"
|
||||
)
|
||||
response = await self._client.aio.models.generate_content(
|
||||
model=self._model,
|
||||
contents=prompt,
|
||||
config=types.GenerateContentConfig(
|
||||
temperature=0, response_mime_type="application/json"
|
||||
),
|
||||
)
|
||||
return parse_scores(response.text or "", questions)
|
||||
|
||||
|
||||
def parse_scores(text: str, questions: Mapping[str, Any]) -> dict[str, float]:
|
||||
"""The judge's reply as scores; a missing or malformed answer scores 0."""
|
||||
try:
|
||||
raw = json.loads(text)
|
||||
except ValueError:
|
||||
raw = {}
|
||||
if not isinstance(raw, dict):
|
||||
raw = {}
|
||||
scores = {}
|
||||
for key in questions:
|
||||
value = raw.get(key, 0)
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
value = 1.0 if value is True else 0.0
|
||||
scores[key] = min(1.0, max(0.0, float(value)))
|
||||
return scores
|
||||
Reference in New Issue
Block a user