103 lines
3.3 KiB
Python
103 lines
3.3 KiB
Python
"""Every Execution is judged as it ends (iteration 5 step 19)."""
|
|
|
|
import json
|
|
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from execution_engine.adk.fake_runtime import FakeAgentRuntime, FakeDecider
|
|
from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec
|
|
from execution_engine.agent import ExecutionEngine
|
|
from execution_engine.nodes.router import Router, Thresholds
|
|
from execution_engine.recorder import InMemoryRecorder
|
|
from tests.reference_workflow.assessment import QUALITY_RUBRIC, assess
|
|
from tests.reference_workflow.workflow import (
|
|
DEPENDENCY_GRAPH,
|
|
WORKFLOW,
|
|
sample_submission,
|
|
)
|
|
|
|
ROLES = {
|
|
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
|
**{
|
|
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
ADVICE = {"design_recommendations": {"content": "Use two zones for the database."}}
|
|
|
|
|
|
def graded(level: float) -> FakeDecider:
|
|
return FakeDecider({key: Answer(level, 0.9) for key in QUALITY_RUBRIC})
|
|
|
|
|
|
async def test_the_rubric_is_scored_from_nought_to_one_with_its_mean():
|
|
decider = graded(3.0)
|
|
decider.answers["specific"] = Answer(1.5, 0.9)
|
|
|
|
assessment = await assess(decider, ADVICE)
|
|
|
|
assert assessment == {
|
|
"recommendations_quality": {
|
|
"specific": 0.5,
|
|
"actionable": 1.0,
|
|
"trade_offs": 1.0,
|
|
"traceable": 1.0,
|
|
},
|
|
"mean": 0.88,
|
|
"meets_bar": True,
|
|
}
|
|
document, _, _ = decider.asked[0]
|
|
assert document == "Use two zones for the database."
|
|
|
|
|
|
async def test_a_weak_design_is_reported_as_below_the_bar():
|
|
assessment = await assess(graded(1.0), ADVICE)
|
|
|
|
assert assessment["mean"] == 0.33 and assessment["meets_bar"] is False
|
|
|
|
|
|
async def test_nothing_to_judge_or_no_decision_gives_no_assessment():
|
|
assert await assess(graded(3.0), {}) is None
|
|
assert await assess(NoDecider(), ADVICE) is None
|
|
|
|
|
|
def engine(tmp_path, decider, **options) -> ExecutionEngine:
|
|
return ExecutionEngine(
|
|
workflow=WORKFLOW,
|
|
runtime=FakeAgentRuntime(
|
|
{
|
|
spec.specialist: [[sample_submission(kind)]]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
),
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
decider=decider,
|
|
**options,
|
|
)
|
|
|
|
|
|
async def test_a_completed_execution_carries_its_assessment(tmp_path):
|
|
result = await engine(tmp_path, graded(1.0), assess_at_end=True).start(
|
|
"Ingest webhooks", pre_approved=True
|
|
)
|
|
|
|
# A low score is reported and the Execution still completes.
|
|
assert result.status == "completed"
|
|
assert result.state["assessment"]["meets_bar"] is False
|
|
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
|
|
assert manifest["assessment"] == result.state["assessment"]
|
|
|
|
|
|
async def test_an_execution_is_not_judged_unless_that_is_switched_on(tmp_path):
|
|
decider = graded(3.0)
|
|
|
|
result = await engine(tmp_path, decider).start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.state["assessment"] is None and decider.asked == []
|
|
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
|
|
assert manifest["assessment"] is None
|