feat: establish the Target architecture as a graph before code
Design is now approved before any infrastructure code is written (ADR-0005). A new target-architect Specialist produces one graph of Elements, Relationships and Checks; the diagram is rendered from it, deployment guidance must account for every Check, and validation gives a verdict per Check. Requirements, components, products and recommendations carry ids in data files, and a submission with a gap in the Requirements chain is refused. Also in this commit: - Customer cases group Executions, with an intent history per case - lesson candidates are captured when an Execution ends - evaluation gains chain, verdict, judged and interactive checks, and per-case thresholds - Terraform providers are cached between checks - the requirements analyst proposes defaults instead of asking - the legacy-baseline case and the checks that only read it are removed - ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed - plans for iterations 4 to 7 under .agents/plans/ Unproven with a real model: no evaluation run has completed on this design. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,8 +2,10 @@
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.adk.fake_runtime import SAMPLE_DATA
|
||||
from app.adk.runtime import ToolResult
|
||||
from app.agent import Interaction
|
||||
from app.workflows.target_architecture import parse_graph, render_mermaid
|
||||
from eval.scoring import CaseSpec, score_execution, template_headings
|
||||
|
||||
TEMPLATE = (
|
||||
@@ -25,11 +27,18 @@ def guide(headings=HEADINGS) -> str:
|
||||
return "\n".join(f"## {heading}\nContent." for heading in headings)
|
||||
|
||||
|
||||
GRAPH = SAMPLE_DATA["target-architecture.json"]
|
||||
|
||||
|
||||
def execution(status="completed", **overrides) -> Interaction:
|
||||
deliverables = {
|
||||
"requirements": {
|
||||
"content": "## Functional requirements\nAccept webhooks.\n## Current state\nNone.",
|
||||
"attachments": {},
|
||||
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]},
|
||||
},
|
||||
"technical_decomposition": {
|
||||
"content": "## Components\nIngress.",
|
||||
"attachments": {"components.json": SAMPLE_DATA["components.json"]},
|
||||
},
|
||||
"product_selection": {
|
||||
"content": (
|
||||
@@ -39,12 +48,19 @@ def execution(status="completed", **overrides) -> Interaction:
|
||||
"**Pros**: control **Cons**: toil |\n"
|
||||
"| Queue | Pub/Sub | ... | Cloud Tasks | ... |\n| Archive | Cloud Storage | ... |"
|
||||
),
|
||||
"attachments": {},
|
||||
"attachments": {"products.json": SAMPLE_DATA["products.json"]},
|
||||
},
|
||||
"architecture_diagram": {
|
||||
"content": "diagram",
|
||||
"design_recommendations": {
|
||||
"content": "## Reliability\nUse two zones because NFR-1 asks for 99.9%.",
|
||||
"attachments": {
|
||||
"architecture.mmd": "flowchart TD\n A[Client] --> B[Run]\n"
|
||||
"recommendations.json": SAMPLE_DATA["recommendations.json"]
|
||||
},
|
||||
},
|
||||
"target_architecture": {
|
||||
"content": "## Elements\n## Relationships\n## Checks\n",
|
||||
"attachments": {
|
||||
"target-architecture.json": GRAPH,
|
||||
"architecture.mmd": render_mermaid(parse_graph(GRAPH)),
|
||||
},
|
||||
},
|
||||
"deployment_guidance": {
|
||||
@@ -52,8 +68,13 @@ def execution(status="completed", **overrides) -> Interaction:
|
||||
"attachments": {
|
||||
"terraform/main.tf": "resource {}",
|
||||
"terraform/variables.tf": "",
|
||||
"implementation.json": SAMPLE_DATA["implementation.json"],
|
||||
},
|
||||
},
|
||||
"validation_results": {
|
||||
"content": "## Deployment dry-run\nPass.",
|
||||
"attachments": {"verdicts.json": SAMPLE_DATA["verdicts.json"]},
|
||||
},
|
||||
"solution_guide": {"content": guide(), "attachments": {}},
|
||||
}
|
||||
for kind, value in overrides.items():
|
||||
@@ -148,7 +169,7 @@ async def test_invalid_terraform_and_diagram_fail_and_warnings_fail_completion()
|
||||
|
||||
result = await score_execution(
|
||||
CASE,
|
||||
execution("completed_with_warnings", architecture_diagram=diagram),
|
||||
execution("completed_with_warnings", target_architecture=diagram),
|
||||
HEADINGS,
|
||||
failing_terraform,
|
||||
)
|
||||
@@ -181,3 +202,276 @@ def test_benchmark_cases_load_with_valid_answer_patterns():
|
||||
for pattern, answer in case.scripted_answers:
|
||||
re.compile(pattern)
|
||||
assert answer
|
||||
|
||||
|
||||
# --- checks that can see quality (iteration 4, step 8) ---------------------------------
|
||||
|
||||
|
||||
class FakeJudge:
|
||||
"""Scores every question with a fixed value, or per id from a mapping."""
|
||||
|
||||
def __init__(self, default=1.0, **by_id):
|
||||
self.default, self.by_id = default, by_id
|
||||
self.asked: list[dict] = []
|
||||
|
||||
async def answer(self, document, questions):
|
||||
self.asked.append(dict(questions))
|
||||
return {key: self.by_id.get(key, self.default) for key in questions}
|
||||
|
||||
|
||||
CHECKLISTS = {
|
||||
"reliability": ["No single point of failure.", "Backups are tested."],
|
||||
"security": ["Least privilege is applied."],
|
||||
}
|
||||
|
||||
|
||||
def deliverable(content: str, **attachments: str) -> dict:
|
||||
return {"content": content, "attachments": attachments}
|
||||
|
||||
|
||||
async def test_a_requirement_that_reaches_no_check_breaks_the_chain():
|
||||
import json
|
||||
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
requirements.append({"id": "NFR-2", "kind": "non_functional", "text": "Retain."})
|
||||
|
||||
result = await score_execution(
|
||||
CASE,
|
||||
execution(
|
||||
requirements=deliverable(
|
||||
"r", **{"requirements.json": json.dumps(requirements)}
|
||||
)
|
||||
),
|
||||
HEADINGS,
|
||||
passing_terraform,
|
||||
)
|
||||
|
||||
chain = check(result, "requirements_chain_complete")
|
||||
assert not chain.passed
|
||||
assert chain.detail == "Requirement NFR-2 has no Check"
|
||||
|
||||
|
||||
async def test_a_check_left_failed_or_unverifiable_fails_the_case():
|
||||
import json
|
||||
|
||||
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
verdicts["checks"][0]["verdict"] = "fail"
|
||||
verdicts["checks"][1]["verdict"] = "not_verifiable"
|
||||
verdicts["proposed_additions"] = [
|
||||
{"statement": "s", "applies_to": "E-1", "reason": "NFR-1"}
|
||||
]
|
||||
|
||||
result = await score_execution(
|
||||
CASE,
|
||||
execution(
|
||||
validation_results=deliverable(
|
||||
"v", **{"verdicts.json": json.dumps(verdicts)}
|
||||
)
|
||||
),
|
||||
HEADINGS,
|
||||
passing_terraform,
|
||||
)
|
||||
|
||||
assert (
|
||||
check(result, "every_check_passes").detail == "K-1 failed; K-2 not verifiable"
|
||||
)
|
||||
|
||||
|
||||
async def test_a_diagram_that_is_not_the_graphs_fails():
|
||||
drawn = deliverable(
|
||||
"t",
|
||||
**{
|
||||
"target-architecture.json": GRAPH,
|
||||
"architecture.mmd": "flowchart LR\n a --> b\n",
|
||||
},
|
||||
)
|
||||
|
||||
result = await score_execution(
|
||||
CASE, execution(target_architecture=drawn), HEADINGS, passing_terraform
|
||||
)
|
||||
|
||||
assert not check(result, "diagram_matches_graph").passed
|
||||
assert check(await _complete(), "diagram_matches_graph").passed
|
||||
|
||||
|
||||
async def _complete():
|
||||
return await score_execution(CASE, execution(), HEADINGS, passing_terraform)
|
||||
|
||||
|
||||
async def test_judged_checks_run_only_with_a_judge():
|
||||
without = await score_execution(CASE, execution(), HEADINGS, passing_terraform)
|
||||
judged = await score_execution(
|
||||
CASE,
|
||||
execution(),
|
||||
HEADINGS,
|
||||
passing_terraform,
|
||||
judge=FakeJudge(),
|
||||
pillar_checklists=CHECKLISTS,
|
||||
)
|
||||
|
||||
names = {c.name for c in judged.checks} - {c.name for c in without.checks}
|
||||
assert names == {"pillar_checklists_covered", "recommendations_quality"}
|
||||
assert judged.score == 1.0
|
||||
|
||||
|
||||
async def test_checklist_coverage_is_judged_per_pillar_and_the_weakest_decides():
|
||||
judge = FakeJudge(**{"reliability.1": 0.0})
|
||||
|
||||
result = await score_execution(
|
||||
CASE,
|
||||
execution(),
|
||||
HEADINGS,
|
||||
passing_terraform,
|
||||
judge=judge,
|
||||
pillar_checklists=CHECKLISTS,
|
||||
)
|
||||
|
||||
coverage = check(result, "pillar_checklists_covered")
|
||||
assert not coverage.passed
|
||||
assert coverage.detail == "reliability 50%, security 100%"
|
||||
assert "Backups are tested." in judge.asked[0]["reliability.1"]["text"]
|
||||
|
||||
|
||||
async def test_generic_recommendations_fail_the_quality_rubric():
|
||||
result = await score_execution(
|
||||
CASE,
|
||||
execution(),
|
||||
HEADINGS,
|
||||
passing_terraform,
|
||||
judge=FakeJudge(default=1.0, specific=0.0, actionable=0.33, traceable=0.33),
|
||||
pillar_checklists=CHECKLISTS,
|
||||
)
|
||||
|
||||
quality = check(result, "recommendations_quality")
|
||||
assert not quality.passed
|
||||
assert quality.detail.startswith("mean 0.42")
|
||||
|
||||
|
||||
async def test_an_interactive_case_checks_specialists_ask_before_submitting():
|
||||
from dataclasses import replace
|
||||
|
||||
interactive = replace(CASE, interactive=True)
|
||||
asked = [
|
||||
{"deliverable": "requirements", "outcome": "NeedsInput"},
|
||||
{"deliverable": "requirements", "outcome": "Submitted"},
|
||||
{"deliverable": None, "outcome": "TurnEnded"},
|
||||
]
|
||||
silent = [*asked, {"deliverable": "product_selection", "outcome": "Submitted"}]
|
||||
|
||||
good = await score_execution(
|
||||
interactive, execution(), HEADINGS, passing_terraform, timeline=asked
|
||||
)
|
||||
bad = await score_execution(
|
||||
interactive, execution(), HEADINGS, passing_terraform, timeline=silent
|
||||
)
|
||||
batch = await score_execution(CASE, execution(), HEADINGS, passing_terraform)
|
||||
|
||||
assert check(good, "specialists_ask_before_submitting").passed
|
||||
assert (
|
||||
check(bad, "specialists_ask_before_submitting").detail
|
||||
== "submitted without asking: product_selection"
|
||||
)
|
||||
assert "specialists_ask_before_submitting" not in {c.name for c in batch.checks}
|
||||
|
||||
|
||||
def test_exceeding_a_threshold_is_a_regression():
|
||||
from dataclasses import replace
|
||||
|
||||
from eval.scoring import regressions
|
||||
|
||||
case = replace(CASE, thresholds={"revisions": 2, "seconds": 3000, "tokens": 4e6})
|
||||
|
||||
assert regressions(case, {"revisions": 2, "seconds": 2999.5, "tokens": 3.9e6}) == []
|
||||
assert regressions(case, {"revisions": 3, "seconds": 3100, "tokens": None}) == [
|
||||
"revisions: 3 > 2",
|
||||
"seconds: 3100 > 3000",
|
||||
]
|
||||
assert regressions(CASE, {"revisions": 9}) == []
|
||||
|
||||
|
||||
def test_every_pillar_skill_yields_checklist_items():
|
||||
from app.providers import load_provider
|
||||
from eval.scoring import checklist_items
|
||||
|
||||
bundle = load_provider("gcp")
|
||||
|
||||
for pillar, skill in bundle.pillar_skills.items():
|
||||
items = checklist_items(bundle.skills[skill].path / "SKILL.md")
|
||||
assert len(items) >= 3, pillar
|
||||
assert all(
|
||||
not item.startswith(("-", "*")) and "\n" not in item for item in items
|
||||
)
|
||||
|
||||
|
||||
def test_a_judge_reply_is_parsed_defensively():
|
||||
from eval.judge import parse_scores
|
||||
|
||||
questions = {"a": {}, "b": {}, "c": {}, "d": {}}
|
||||
|
||||
assert parse_scores('{"a": 1, "b": 0.33, "c": true, "d": 7}', questions) == {
|
||||
"a": 1.0,
|
||||
"b": 0.33,
|
||||
"c": 1.0,
|
||||
"d": 1.0,
|
||||
}
|
||||
assert parse_scores("not json", questions) == dict.fromkeys(questions, 0.0)
|
||||
assert parse_scores('{"a": "yes"}', questions)["a"] == 0.0
|
||||
|
||||
|
||||
async def test_a_scripted_user_drives_an_interactive_execution(tmp_path):
|
||||
from dataclasses import replace
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
|
||||
from app.adk.runtime import RoleSpec
|
||||
from app.agent import ExecutionEngine
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
||||
from eval.solution_eval import run_with_scripted_user
|
||||
|
||||
requirements = DeliverableKind.REQUIREMENTS
|
||||
analyst = DEPENDENCY_GRAPH[requirements].specialist
|
||||
roles = {
|
||||
"orchestrator": RoleSpec("orchestrator", "", budget=10),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "", budget=10)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
[Call("delegate", {"kinds": ["requirements"]})],
|
||||
[Call("ask_user")],
|
||||
[Call("delegate", {"kinds": ["requirements"]})],
|
||||
[Call("ask_user", {"questions": ["Which region?"]})],
|
||||
[Call("ask_user", {"questions": ["Anything else?"]})],
|
||||
],
|
||||
analyst: [
|
||||
[Call("needs_input", {"questions": ["Do you approve this draft?"]})],
|
||||
[sample_submission(requirements)],
|
||||
],
|
||||
}
|
||||
)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
)
|
||||
case = replace(
|
||||
CASE,
|
||||
interactive=True,
|
||||
scripted_answers=(("approve", "Approved."), ("region", "europe-west2.")),
|
||||
)
|
||||
|
||||
final = await run_with_scripted_user(
|
||||
engine, case, "exec-interactive", max_user_turns=2
|
||||
)
|
||||
|
||||
replies = [m["text"] for m in final.state["conversation"] if m["from"] == "user"]
|
||||
assert any("Approved." in reply for reply in replies)
|
||||
assert any("europe-west2." in reply for reply in replies)
|
||||
assert final.state["deliverables"]["requirements"]["content"]
|
||||
|
||||
Reference in New Issue
Block a user