feat: working orhcestration engine
This commit is contained in:
172
tests/execution/test_intake.py
Normal file
172
tests/execution/test_intake.py
Normal file
@@ -0,0 +1,172 @@
|
||||
"""Intake: what a message that starts something is asking for."""
|
||||
|
||||
import json
|
||||
|
||||
from execution_engine.adk.fake_runtime import FakeDecider
|
||||
from execution_engine.adk.runtime import Answer, NoDecider
|
||||
from execution_engine.nodes.intake import CHANGE_REQUEST, NEW_REQUEST, QUESTION, Intake
|
||||
from tests.reference_workflow.workflow import WORKFLOW
|
||||
|
||||
|
||||
def delivered(tmp_path, case: str, statuses: list[str]) -> None:
|
||||
folder = tmp_path / "deliverables" / "cases" / case / "intent"
|
||||
folder.mkdir(parents=True)
|
||||
intents = [
|
||||
{
|
||||
"version": n,
|
||||
"workflow_request": f"Design thing {n}.",
|
||||
"status": status,
|
||||
"execution_id": f"v{n}",
|
||||
}
|
||||
for n, status in enumerate(statuses, start=1)
|
||||
]
|
||||
(folder / "history.json").write_text(
|
||||
json.dumps({"customer_case": case, "intents": intents})
|
||||
)
|
||||
|
||||
|
||||
async def test_without_a_decision_every_message_is_a_new_request(tmp_path):
|
||||
reading = await Intake(NoDecider(), tmp_path).read("What can you do?")
|
||||
|
||||
assert reading.kind == NEW_REQUEST and not reading.judged
|
||||
|
||||
|
||||
async def test_a_confident_question_is_read_as_one(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer(QUESTION, 0.98)})
|
||||
|
||||
reading = await Intake(decider, tmp_path).read("How long does it take?", scope="t1")
|
||||
|
||||
state, questions, scope = decider.asked[0]
|
||||
assert reading.kind == QUESTION and reading.judged
|
||||
assert scope == "t1"
|
||||
assert questions["intake"].options == (NEW_REQUEST, CHANGE_REQUEST, QUESTION)
|
||||
assert state.endswith("The user's message: How long does it take?")
|
||||
|
||||
|
||||
async def test_a_reading_that_is_not_sure_enough_starts_new_work(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer(QUESTION, 0.6)})
|
||||
|
||||
assert (await Intake(decider, tmp_path, act=0.75).read("hello")).kind == NEW_REQUEST
|
||||
|
||||
|
||||
async def test_a_change_needs_something_delivered_to_change(tmp_path):
|
||||
decider = FakeDecider({"intake": Answer(CHANGE_REQUEST, 0.99)})
|
||||
delivered(tmp_path, "acme", ["completed", "in_progress"])
|
||||
intake = Intake(decider, tmp_path)
|
||||
|
||||
with_history = await intake.read("Raise availability to 99.99%.", "acme")
|
||||
without = await intake.read("Raise availability to 99.99%.", "new-client")
|
||||
no_case = await intake.read("Raise availability to 99.99%.")
|
||||
|
||||
assert with_history.kind == CHANGE_REQUEST and with_history.earlier == 1
|
||||
assert with_history.latest == "v1" # what the change would change
|
||||
assert "- Design thing 1." in decider.asked[0][0]
|
||||
assert "Design thing 2." not in decider.asked[0][0] # still in progress
|
||||
assert without.kind == NEW_REQUEST and no_case.kind == NEW_REQUEST
|
||||
|
||||
|
||||
# --- a question about what was delivered ------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_question_about_delivered_work_is_told_from_one_about_the_service(
|
||||
tmp_path,
|
||||
):
|
||||
delivered(tmp_path, "acme", ["completed"])
|
||||
about = FakeDecider(
|
||||
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.9, 0.9)}
|
||||
)
|
||||
general = FakeDecider(
|
||||
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.1, 0.9)}
|
||||
)
|
||||
|
||||
first = await Intake(about, tmp_path).read("Why Cloud SQL?", "acme")
|
||||
second = await Intake(general, tmp_path).read("What can you do?", "acme")
|
||||
|
||||
assert first.kind == QUESTION and first.about_delivered and first.latest == "v1"
|
||||
assert second.kind == QUESTION and not second.about_delivered
|
||||
|
||||
|
||||
async def test_with_nothing_delivered_a_question_is_never_about_delivered_work(
|
||||
tmp_path,
|
||||
):
|
||||
decider = FakeDecider(
|
||||
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.99, 0.9)}
|
||||
)
|
||||
|
||||
reading = await Intake(decider, tmp_path).read("Why Cloud SQL?", "new-client")
|
||||
|
||||
assert not reading.about_delivered
|
||||
assert len(decider.asked) == 1 # the second question was not asked
|
||||
|
||||
|
||||
async def test_an_answer_is_written_from_the_delivered_guide_and_nothing_else(
|
||||
tmp_path,
|
||||
):
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from execution_engine.adk.fake_runtime import (
|
||||
Call,
|
||||
FakeAgentRuntime,
|
||||
)
|
||||
from execution_engine.adk.runtime import RoleSpec
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.nodes.intake import DeliveredAnswers
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from tests.reference_workflow.workflow import (
|
||||
DEPENDENCY_GRAPH,
|
||||
sample_submission,
|
||||
)
|
||||
|
||||
roles = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
seen: dict = {}
|
||||
|
||||
def writer(task):
|
||||
seen.update(task.context)
|
||||
return [Call("write_reply", {"text": "Cloud SQL, for relational bookings."})]
|
||||
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
**{
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
},
|
||||
"reply-writer": [writer],
|
||||
}
|
||||
)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
await engine.start(
|
||||
"Design bookings.", customer_case="acme", pre_approved=True, execution_id="v1"
|
||||
)
|
||||
runs_before = len(runtime.runs)
|
||||
answers = DeliveredAnswers(
|
||||
runtime,
|
||||
RoleSpec("reply-writer", "Write.", budget=3),
|
||||
tmp_path,
|
||||
"solution_guide",
|
||||
)
|
||||
|
||||
answer = await answers.answer("Why Cloud SQL?", "acme", "v1")
|
||||
|
||||
assert answer == "Cloud SQL, for relational bookings."
|
||||
assert seen["question"] == "Why Cloud SQL?"
|
||||
assert seen["delivered"]["execution_id"] == "v1"
|
||||
assert seen["delivered"]["request"] == "Design bookings."
|
||||
assert seen["delivered"]["final_document"]
|
||||
assert len(runtime.runs) == runs_before + 1 # one writer run; no Execution
|
||||
# An Execution that is not there gives no answer, not an error.
|
||||
assert await answers.answer("Why?", "acme", "missing") is None
|
||||
Reference in New Issue
Block a user