chore: housekeeping from the runs of 2026-10-09
Some checks failed
validation / verify (push) Failing after 12s
Some checks failed
validation / verify (push) Failing after 12s
- An Execution's files are written as each Deliverable is submitted, marked in_progress, so a killed process or one waiting for its user leaves them on disk. A write removes files an earlier write left that it no longer has. - A question about delivered work is answered from that Execution's solution guide by the reply writer; a question about the service is answered as before. Nothing is started for either. - Lessons: prove writes only for lessons the Skill holds; the Skill records the vendored Skill versions the lessons were read against, status names any that changed, and a test fails until they are reviewed. - Four more code patterns: Firestore, BigQuery, Eventarc and GKE. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -75,7 +75,7 @@ def complete_run_turns() -> dict:
|
||||
|
||||
|
||||
def client_for(
|
||||
tmp_path: Path, turns: dict, decider=None
|
||||
tmp_path: Path, turns: dict, decider=None, answers=None
|
||||
) -> tuple[TestClient, FakeAgentRuntime]:
|
||||
bundle = load_provider("gcp")
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
@@ -96,6 +96,7 @@ def client_for(
|
||||
roles,
|
||||
{"compute": "API not enabled"},
|
||||
intake=Intake(decider, tmp_path) if decider else None,
|
||||
answers=answers,
|
||||
)
|
||||
|
||||
return TestClient(create_app(get_settings(), fake_assembly)), runtime
|
||||
@@ -398,6 +399,74 @@ def delivered_case(tmp_path: Path, case: str = "acme") -> None:
|
||||
)
|
||||
|
||||
|
||||
class Answers:
|
||||
"""Stands in for the reply writer answering from delivered files."""
|
||||
|
||||
def __init__(self, reply):
|
||||
self.reply = reply
|
||||
self.asked: list[tuple[str, str, str]] = []
|
||||
|
||||
async def answer(self, question, customer_case, execution_id):
|
||||
self.asked.append((question, customer_case, execution_id))
|
||||
return self.reply
|
||||
|
||||
|
||||
ABOUT_DELIVERED = {
|
||||
"intake": Answer("question", 0.98),
|
||||
"about_delivered": Answer(0.92, 0.9),
|
||||
}
|
||||
|
||||
|
||||
def test_a_question_about_delivered_work_is_answered_from_it(tmp_path):
|
||||
delivered_case(tmp_path)
|
||||
answers = Answers("Cloud SQL, because bookings are relational.")
|
||||
client, runtime = client_for(tmp_path, {}, FakeDecider(ABOUT_DELIVERED), answers)
|
||||
with client:
|
||||
task = send(
|
||||
client,
|
||||
[{"text": "Why did you choose Cloud SQL?"}],
|
||||
metadata={"customer_case": "acme"},
|
||||
)
|
||||
|
||||
text, data = status_parts(task)
|
||||
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
|
||||
assert text == "Cloud SQL, because bookings are relational."
|
||||
assert data == {"status": "answered", "intake": "question", "about": "v1"}
|
||||
assert answers.asked == [("Why did you choose Cloud SQL?", "acme", "v1")]
|
||||
assert runtime.runs == []
|
||||
|
||||
|
||||
def test_a_question_that_could_not_be_answered_says_where_to_look(tmp_path):
|
||||
delivered_case(tmp_path)
|
||||
client, _ = client_for(tmp_path, {}, FakeDecider(ABOUT_DELIVERED), Answers(None))
|
||||
with client:
|
||||
task = send(
|
||||
client, [{"text": "Why Cloud SQL?"}], metadata={"customer_case": "acme"}
|
||||
)
|
||||
|
||||
text, data = status_parts(task)
|
||||
assert "could not answer that from the documents delivered in Execution v1" in text
|
||||
assert data["about"] == "v1"
|
||||
|
||||
|
||||
def test_a_question_about_the_service_is_still_answered_in_general(tmp_path):
|
||||
delivered_case(tmp_path)
|
||||
decider = FakeDecider(
|
||||
{"intake": Answer("question", 0.98), "about_delivered": Answer(0.08, 0.9)}
|
||||
)
|
||||
answers = Answers("unused")
|
||||
client, _ = client_for(tmp_path, {}, decider, answers)
|
||||
with client:
|
||||
task = send(
|
||||
client, [{"text": "What can you do?"}], metadata={"customer_case": "acme"}
|
||||
)
|
||||
|
||||
text, data = status_parts(task)
|
||||
assert text.startswith("I design cloud solution architectures.")
|
||||
assert data == {"status": "answered", "intake": "question"}
|
||||
assert answers.asked == []
|
||||
|
||||
|
||||
def test_a_change_to_delivered_work_names_the_baseline_and_starts_nothing(tmp_path):
|
||||
delivered_case(tmp_path)
|
||||
decider = FakeDecider({"intake": Answer("change_request", 0.98)})
|
||||
|
||||
@@ -62,3 +62,100 @@ async def test_a_change_needs_something_delivered_to_change(tmp_path):
|
||||
assert "- Design thing 1." in decider.asked[0][0]
|
||||
assert "Design thing 2." not in decider.asked[0][0] # still in progress
|
||||
assert without.kind == NEW_REQUEST and no_case.kind == NEW_REQUEST
|
||||
|
||||
|
||||
# --- a question about what was delivered ------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_question_about_delivered_work_is_told_from_one_about_the_service(
|
||||
tmp_path,
|
||||
):
|
||||
delivered(tmp_path, "acme", ["completed"])
|
||||
about = FakeDecider(
|
||||
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.9, 0.9)}
|
||||
)
|
||||
general = FakeDecider(
|
||||
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.1, 0.9)}
|
||||
)
|
||||
|
||||
first = await Intake(about, tmp_path).read("Why Cloud SQL?", "acme")
|
||||
second = await Intake(general, tmp_path).read("What can you do?", "acme")
|
||||
|
||||
assert first.kind == QUESTION and first.about_delivered and first.latest == "v1"
|
||||
assert second.kind == QUESTION and not second.about_delivered
|
||||
|
||||
|
||||
async def test_with_nothing_delivered_a_question_is_never_about_delivered_work(
|
||||
tmp_path,
|
||||
):
|
||||
decider = FakeDecider(
|
||||
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.99, 0.9)}
|
||||
)
|
||||
|
||||
reading = await Intake(decider, tmp_path).read("Why Cloud SQL?", "new-client")
|
||||
|
||||
assert not reading.about_delivered
|
||||
assert len(decider.asked) == 1 # the second question was not asked
|
||||
|
||||
|
||||
async def test_an_answer_is_written_from_the_delivered_guide_and_nothing_else(
|
||||
tmp_path,
|
||||
):
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
|
||||
from app.adk.runtime import RoleSpec
|
||||
from app.agent import ExecutionEngine
|
||||
from app.nodes.intake import DeliveredAnswers
|
||||
from app.nodes.router import Router, Thresholds
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH
|
||||
|
||||
roles = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
seen: dict = {}
|
||||
|
||||
def writer(task):
|
||||
seen.update(task.context)
|
||||
return [Call("write_reply", {"text": "Cloud SQL, for relational bookings."})]
|
||||
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
**{
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
},
|
||||
"reply-writer": [writer],
|
||||
}
|
||||
)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
await engine.start(
|
||||
"Design bookings.", customer_case="acme", pre_approved=True, execution_id="v1"
|
||||
)
|
||||
runs_before = len(runtime.runs)
|
||||
answers = DeliveredAnswers(
|
||||
runtime, RoleSpec("reply-writer", "Write.", budget=3), tmp_path
|
||||
)
|
||||
|
||||
answer = await answers.answer("Why Cloud SQL?", "acme", "v1")
|
||||
|
||||
assert answer == "Cloud SQL, for relational bookings."
|
||||
assert seen["question"] == "Why Cloud SQL?"
|
||||
assert seen["delivered"]["execution_id"] == "v1"
|
||||
assert seen["delivered"]["request"] == "Design bookings."
|
||||
assert seen["delivered"]["solution_guide"]
|
||||
assert len(runtime.runs) == runs_before + 1 # one writer run; no Execution
|
||||
# An Execution that is not there gives no answer, not an error.
|
||||
assert await answers.answer("Why?", "acme", "missing") is None
|
||||
|
||||
100
tests/execution/test_outputs_as_it_goes.py
Normal file
100
tests/execution/test_outputs_as_it_goes.py
Normal file
@@ -0,0 +1,100 @@
|
||||
"""An Execution's files are on disk as soon as each Deliverable is submitted.
|
||||
|
||||
Before this they were written only when an Execution ended, so a process that was
|
||||
killed, or an Execution left waiting for its user, had nothing to show.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import FakeAgentRuntime, sample_submission
|
||||
from app.adk.runtime import NoDecider, RoleSpec
|
||||
from app.agent import ExecutionEngine
|
||||
from app.baseline import BaselineError, load_baseline
|
||||
from app.nodes.execution_nodes import _write_outputs
|
||||
from app.nodes.router import Router, Thresholds
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.states.ledger import Ledger
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def folder(tmp_path, execution_id: str):
|
||||
return tmp_path / "deliverables" / "cases" / "acme" / "executions" / execution_id
|
||||
|
||||
|
||||
async def test_an_execution_waiting_for_its_user_has_written_what_exists(tmp_path):
|
||||
engine = ExecutionEngine(
|
||||
runtime=FakeAgentRuntime(
|
||||
{
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
),
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
)
|
||||
|
||||
waiting = await engine.start("Ingest", customer_case="acme", execution_id="v1")
|
||||
|
||||
# It stopped for the first Approval: only the first Phase has been produced.
|
||||
assert waiting.status == "awaiting_user"
|
||||
out = folder(tmp_path, "v1")
|
||||
manifest = json.loads((out / "execution.json").read_text())
|
||||
assert manifest["status"] == "in_progress"
|
||||
assert set(manifest["deliverables"]) == {"requirements", "technical_decomposition"}
|
||||
assert (out / "deliverables" / "requirements.md").read_text()
|
||||
assert (out / "requirements.json").is_file()
|
||||
assert manifest["solution_guide"] is None
|
||||
# Unfinished work is never something a Change request can start from.
|
||||
try:
|
||||
load_baseline(tmp_path, "acme", "v1")
|
||||
except BaselineError as refused:
|
||||
assert "did not complete (in_progress)" in str(refused)
|
||||
else:
|
||||
raise AssertionError("an in-progress Execution was accepted as a Baseline")
|
||||
|
||||
|
||||
def test_a_file_an_earlier_write_left_is_removed_with_the_revision(tmp_path):
|
||||
state = {
|
||||
"execution_id": "v1",
|
||||
"customer_case": "acme",
|
||||
"workflow_request": "Ingest",
|
||||
"deliverables": {
|
||||
"deployment_guidance": {
|
||||
"content": "guide",
|
||||
"attachments": {
|
||||
"terraform/main.tf": "one",
|
||||
"terraform/queue.tf": "two",
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
out = folder(tmp_path, "v1")
|
||||
|
||||
_write_outputs(tmp_path, state, "in_progress", Ledger())
|
||||
assert (out / "terraform" / "queue.tf").is_file()
|
||||
(out / "notes.txt").write_text("a person's own file")
|
||||
|
||||
# The Revision folded the queue into main.tf.
|
||||
state["deliverables"]["deployment_guidance"]["attachments"] = {
|
||||
"terraform/main.tf": "one and two"
|
||||
}
|
||||
_write_outputs(tmp_path, state, "completed", Ledger())
|
||||
|
||||
assert not (out / "terraform" / "queue.tf").exists()
|
||||
assert (out / "terraform" / "main.tf").read_text() == "one and two"
|
||||
# Only files this Execution wrote are ever removed.
|
||||
assert (out / "notes.txt").is_file()
|
||||
assert json.loads((out / "execution.json").read_text())["status"] == "completed"
|
||||
@@ -498,16 +498,68 @@ def test_lessons_that_cost_quality_or_save_nothing_are_not_shown_to_help(tmp_pat
|
||||
assert "nothing fell" in render(compare(base, same), base, same)
|
||||
|
||||
|
||||
def test_the_comparison_is_written_into_published_proposals_only(tmp_path):
|
||||
def test_the_comparison_is_written_only_for_lessons_in_the_skill(tmp_path):
|
||||
from app.learning.proof import record
|
||||
|
||||
skill = lessons_skill(tmp_path)
|
||||
issues = tmp_path / "issues"
|
||||
live = publish(approve(proposal(issues), issues), skill, today=DAY)
|
||||
waiting = proposal(issues, title="Another", evidence_key="zzz")
|
||||
gone = publish(
|
||||
approve(proposal(issues, title="Retired", evidence_key="yyy"), issues),
|
||||
skill,
|
||||
today=DAY,
|
||||
)
|
||||
retire(gone.id, skill)
|
||||
|
||||
recorded = record([live, waiting], "| a table |")
|
||||
recorded = record([live, waiting, gone], "| a table |", published(skill))
|
||||
|
||||
assert [p.id for p in recorded] == ["L-01"]
|
||||
assert parse(live.path).proof == "| a table |"
|
||||
assert parse(waiting.path).proof == ""
|
||||
# A retired lesson was not part of what was compared.
|
||||
assert parse(gone.path).proof == ""
|
||||
|
||||
|
||||
# --- lessons are read again when the Skills beside them change ---------------------------
|
||||
|
||||
|
||||
def test_a_changed_upstream_skill_is_flagged_until_the_lessons_are_reviewed(tmp_path):
|
||||
from app.learning.skill import mark_reviewed, upstream_changes
|
||||
|
||||
skill = lessons_skill(tmp_path)
|
||||
issues = tmp_path / "issues"
|
||||
publish(approve(proposal(issues), issues), skill, today=DAY)
|
||||
then = {"architecture": "aaa", "security": "bbb"}
|
||||
mark_reviewed(skill, then)
|
||||
|
||||
assert upstream_changes(skill, then) == []
|
||||
now = {"architecture": "aaa", "security": "ccc", "cost": "ddd"}
|
||||
assert upstream_changes(skill, now) == ["cost", "security"]
|
||||
|
||||
mark_reviewed(skill, now)
|
||||
assert upstream_changes(skill, now) == []
|
||||
|
||||
|
||||
def test_nothing_is_flagged_with_no_lessons_or_nothing_recorded(tmp_path):
|
||||
from app.learning.skill import mark_reviewed, upstream_changes
|
||||
|
||||
skill = lessons_skill(tmp_path)
|
||||
assert upstream_changes(skill, {"architecture": "aaa"}) == []
|
||||
mark_reviewed(skill, {"architecture": "aaa"})
|
||||
# No lesson is published, so there is nothing to review.
|
||||
assert upstream_changes(skill, {"architecture": "zzz"}) == []
|
||||
|
||||
|
||||
def test_the_published_lessons_were_reviewed_against_todays_vendored_skills():
|
||||
"""Syncing the vendored Skills changes what the lessons sit beside. This fails
|
||||
until a person has read each lesson again and run `curate_lessons reviewed`."""
|
||||
from app.learning.skill import upstream_changes, upstream_reviewed
|
||||
from app.providers import load_provider
|
||||
from eval.curate_lessons import upstream_versions
|
||||
|
||||
bundle = load_provider("gcp")
|
||||
skill = bundle.skills[bundle.lessons_skill].path
|
||||
|
||||
assert upstream_reviewed(skill), "run: python -m eval.curate_lessons reviewed"
|
||||
assert upstream_changes(skill, upstream_versions(bundle)) == []
|
||||
|
||||
Reference in New Issue
Block a user