Files
gcp_solution_architecture_a…/tests/execution/test_assessment.py
Jonathan Boniface 567aede5ca feat: overlay Skills, and five more Jev decisions (iterations 4 and 5)
Iteration 4:
- Overlay Skills (step 12): an Execution names an organisation's standards
  Skill from the folders the deployment configures; recorded with its version.
  An example standards Skill and an evaluation case that uses it.
- Lessons (step 10): a prove command that compares a run with lessons against
  one without, and triage of evidence an existing lesson already covers.

Iteration 5:
- Step 8: the references of a Specialist's Skills its task needs are named in
  its context as read_first.
- Step 14: a remote MCP server's reply that instructs the agent is withheld;
  the user's own text is flagged as a risk and nothing more.
- Step 15: in a Change request, items only reworded are listed apart from
  real changes.
- Step 19: each Execution's design recommendations are judged when it ends and
  the scores kept in its record. The evaluation takes its rubric from the same
  place.

Each question was calibrated against live Jev; none has run in a case. Steps
6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 17:45:56 +01:00

98 lines
3.2 KiB
Python

"""Every Execution is judged as it ends (iteration 5 step 19)."""
import json
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import FakeAgentRuntime, FakeDecider, sample_submission
from app.adk.runtime import Answer, NoDecider, RoleSpec
from app.agent import ExecutionEngine
from app.nodes.router import Router, Thresholds
from app.recorder import InMemoryRecorder
from app.tools.assessment import QUALITY_RUBRIC, assess
from app.workflows.deliverables import DEPENDENCY_GRAPH
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
ADVICE = {"design_recommendations": {"content": "Use two zones for the database."}}
def graded(level: float) -> FakeDecider:
return FakeDecider({key: Answer(level, 0.9) for key in QUALITY_RUBRIC})
async def test_the_rubric_is_scored_from_nought_to_one_with_its_mean():
decider = graded(3.0)
decider.answers["specific"] = Answer(1.5, 0.9)
assessment = await assess(decider, ADVICE)
assert assessment == {
"recommendations_quality": {
"specific": 0.5,
"actionable": 1.0,
"trade_offs": 1.0,
"traceable": 1.0,
},
"mean": 0.88,
"meets_bar": True,
}
document, _, _ = decider.asked[0]
assert document == "Use two zones for the database."
async def test_a_weak_design_is_reported_as_below_the_bar():
assessment = await assess(graded(1.0), ADVICE)
assert assessment["mean"] == 0.33 and assessment["meets_bar"] is False
async def test_nothing_to_judge_or_no_decision_gives_no_assessment():
assert await assess(graded(3.0), {}) is None
assert await assess(NoDecider(), ADVICE) is None
def engine(tmp_path, decider, **options) -> ExecutionEngine:
return ExecutionEngine(
runtime=FakeAgentRuntime(
{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
),
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
decider=decider,
**options,
)
async def test_a_completed_execution_carries_its_assessment(tmp_path):
result = await engine(tmp_path, graded(1.0), assess_at_end=True).start(
"Ingest webhooks", pre_approved=True
)
# A low score is reported and the Execution still completes.
assert result.status == "completed"
assert result.state["assessment"]["meets_bar"] is False
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
assert manifest["assessment"] == result.state["assessment"]
async def test_an_execution_is_not_judged_unless_that_is_switched_on(tmp_path):
decider = graded(3.0)
result = await engine(tmp_path, decider).start("Ingest webhooks", pre_approved=True)
assert result.state["assessment"] is None and decider.asked == []
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
assert manifest["assessment"] is None