feat: working orhcestration engine
This commit is contained in:
102
tests/execution/test_assessment.py
Normal file
102
tests/execution/test_assessment.py
Normal file
@@ -0,0 +1,102 @@
|
||||
"""Every Execution is judged as it ends (iteration 5 step 19)."""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import FakeAgentRuntime, FakeDecider
|
||||
from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from tests.reference_workflow.assessment import QUALITY_RUBRIC, assess
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
WORKFLOW,
|
||||
sample_submission,
|
||||
)
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
ADVICE = {"design_recommendations": {"content": "Use two zones for the database."}}
|
||||
|
||||
|
||||
def graded(level: float) -> FakeDecider:
|
||||
return FakeDecider({key: Answer(level, 0.9) for key in QUALITY_RUBRIC})
|
||||
|
||||
|
||||
async def test_the_rubric_is_scored_from_nought_to_one_with_its_mean():
|
||||
decider = graded(3.0)
|
||||
decider.answers["specific"] = Answer(1.5, 0.9)
|
||||
|
||||
assessment = await assess(decider, ADVICE)
|
||||
|
||||
assert assessment == {
|
||||
"recommendations_quality": {
|
||||
"specific": 0.5,
|
||||
"actionable": 1.0,
|
||||
"trade_offs": 1.0,
|
||||
"traceable": 1.0,
|
||||
},
|
||||
"mean": 0.88,
|
||||
"meets_bar": True,
|
||||
}
|
||||
document, _, _ = decider.asked[0]
|
||||
assert document == "Use two zones for the database."
|
||||
|
||||
|
||||
async def test_a_weak_design_is_reported_as_below_the_bar():
|
||||
assessment = await assess(graded(1.0), ADVICE)
|
||||
|
||||
assert assessment["mean"] == 0.33 and assessment["meets_bar"] is False
|
||||
|
||||
|
||||
async def test_nothing_to_judge_or_no_decision_gives_no_assessment():
|
||||
assert await assess(graded(3.0), {}) is None
|
||||
assert await assess(NoDecider(), ADVICE) is None
|
||||
|
||||
|
||||
def engine(tmp_path, decider, **options) -> ExecutionEngine:
|
||||
return ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=FakeAgentRuntime(
|
||||
{
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
),
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
decider=decider,
|
||||
**options,
|
||||
)
|
||||
|
||||
|
||||
async def test_a_completed_execution_carries_its_assessment(tmp_path):
|
||||
result = await engine(tmp_path, graded(1.0), assess_at_end=True).start(
|
||||
"Ingest webhooks", pre_approved=True
|
||||
)
|
||||
|
||||
# A low score is reported and the Execution still completes.
|
||||
assert result.status == "completed"
|
||||
assert result.state["assessment"]["meets_bar"] is False
|
||||
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
|
||||
assert manifest["assessment"] == result.state["assessment"]
|
||||
|
||||
|
||||
async def test_an_execution_is_not_judged_unless_that_is_switched_on(tmp_path):
|
||||
decider = graded(3.0)
|
||||
|
||||
result = await engine(tmp_path, decider).start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.state["assessment"] is None and decider.asked == []
|
||||
manifest = json.loads(next(tmp_path.rglob("execution.json")).read_text())
|
||||
assert manifest["assessment"] is None
|
||||
Reference in New Issue
Block a user