Files
orchestration_engine/tests/execution/test_assessment.py

103 lines
3.3 KiB
Python

"""Every Execution is judged as it ends (iteration 5 step 19)."""
import json
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import FakeAgentRuntime, FakeDecider
from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from tests.reference_workflow.assessment import QUALITY_RUBRIC, assess
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
WORKFLOW,
sample_submission,
)
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
ADVICE = {"design_recommendations": {"content": "Use two zones for the database."}}
def graded(level: float) -> FakeDecider:
return FakeDecider({key: Answer(level, 0.9) for key in QUALITY_RUBRIC})
async def test_the_rubric_is_scored_from_nought_to_one_with_its_mean():
decider = graded(3.0)
decider.answers["specific"] = Answer(1.5, 0.9)
assessment = await assess(decider, ADVICE)
assert assessment == {
"recommendations_quality": {
"specific": 0.5,
"actionable": 1.0,
"trade_offs": 1.0,
"traceable": 1.0,
},
"mean": 0.88,
"meets_bar": True,
}
document, _, _ = decider.asked[0]
assert document == "Use two zones for the database."
async def test_a_weak_design_is_reported_as_below_the_bar():
assessment = await assess(graded(1.0), ADVICE)
assert assessment["mean"] == 0.33 and assessment["meets_bar"] is False
async def test_nothing_to_judge_or_no_decision_gives_no_assessment():
assert await assess(graded(3.0), {}) is None
assert await assess(NoDecider(), ADVICE) is None
def engine(tmp_path, decider, **options) -> ExecutionEngine:
return ExecutionEngine(
workflow=WORKFLOW,
runtime=FakeAgentRuntime(
{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
),
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
decider=decider,
**options,
)
async def test_a_completed_execution_carries_its_assessment(tmp_path):
result = await engine(tmp_path, graded(1.0), assess_at_end=True).start(
"Ingest webhooks", pre_approved=True
)
# A low score is reported and the Execution still completes.
assert result.status == "completed"
assert result.state["assessment"]["meets_bar"] is False
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
assert manifest["assessment"] == result.state["assessment"]
async def test_an_execution_is_not_judged_unless_that_is_switched_on(tmp_path):
decider = graded(3.0)
result = await engine(tmp_path, decider).start("Ingest webhooks", pre_approved=True)
assert result.state["assessment"] is None and decider.asked == []
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
assert manifest["assessment"] is None