chore: housekeeping from the runs of 2026-10-09
Some checks failed
validation / verify (push) Failing after 12s

- An Execution's files are written as each Deliverable is submitted, marked
  in_progress, so a killed process or one waiting for its user leaves them on
  disk. A write removes files an earlier write left that it no longer has.
- A question about delivered work is answered from that Execution's solution
  guide by the reply writer; a question about the service is answered as
  before. Nothing is started for either.
- Lessons: prove writes only for lessons the Skill holds; the Skill records
  the vendored Skill versions the lessons were read against, status names any
  that changed, and a test fails until they are reviewed.
- Four more code patterns: Firestore, BigQuery, Eventarc and GKE.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-10 11:44:38 +01:00
parent 10d23a5f91
commit 71da0ed504
24 changed files with 1109 additions and 43 deletions

View File

@@ -75,7 +75,7 @@ def complete_run_turns() -> dict:
def client_for(
tmp_path: Path, turns: dict, decider=None
tmp_path: Path, turns: dict, decider=None, answers=None
) -> tuple[TestClient, FakeAgentRuntime]:
bundle = load_provider("gcp")
roles = load_roles(AGENTS_DIR, bundle)
@@ -96,6 +96,7 @@ def client_for(
roles,
{"compute": "API not enabled"},
intake=Intake(decider, tmp_path) if decider else None,
answers=answers,
)
return TestClient(create_app(get_settings(), fake_assembly)), runtime
@@ -398,6 +399,74 @@ def delivered_case(tmp_path: Path, case: str = "acme") -> None:
)
class Answers:
"""Stands in for the reply writer answering from delivered files."""
def __init__(self, reply):
self.reply = reply
self.asked: list[tuple[str, str, str]] = []
async def answer(self, question, customer_case, execution_id):
self.asked.append((question, customer_case, execution_id))
return self.reply
ABOUT_DELIVERED = {
"intake": Answer("question", 0.98),
"about_delivered": Answer(0.92, 0.9),
}
def test_a_question_about_delivered_work_is_answered_from_it(tmp_path):
delivered_case(tmp_path)
answers = Answers("Cloud SQL, because bookings are relational.")
client, runtime = client_for(tmp_path, {}, FakeDecider(ABOUT_DELIVERED), answers)
with client:
task = send(
client,
[{"text": "Why did you choose Cloud SQL?"}],
metadata={"customer_case": "acme"},
)
text, data = status_parts(task)
assert task["status"]["state"] == "TASK_STATE_COMPLETED"
assert text == "Cloud SQL, because bookings are relational."
assert data == {"status": "answered", "intake": "question", "about": "v1"}
assert answers.asked == [("Why did you choose Cloud SQL?", "acme", "v1")]
assert runtime.runs == []
def test_a_question_that_could_not_be_answered_says_where_to_look(tmp_path):
delivered_case(tmp_path)
client, _ = client_for(tmp_path, {}, FakeDecider(ABOUT_DELIVERED), Answers(None))
with client:
task = send(
client, [{"text": "Why Cloud SQL?"}], metadata={"customer_case": "acme"}
)
text, data = status_parts(task)
assert "could not answer that from the documents delivered in Execution v1" in text
assert data["about"] == "v1"
def test_a_question_about_the_service_is_still_answered_in_general(tmp_path):
delivered_case(tmp_path)
decider = FakeDecider(
{"intake": Answer("question", 0.98), "about_delivered": Answer(0.08, 0.9)}
)
answers = Answers("unused")
client, _ = client_for(tmp_path, {}, decider, answers)
with client:
task = send(
client, [{"text": "What can you do?"}], metadata={"customer_case": "acme"}
)
text, data = status_parts(task)
assert text.startswith("I design cloud solution architectures.")
assert data == {"status": "answered", "intake": "question"}
assert answers.asked == []
def test_a_change_to_delivered_work_names_the_baseline_and_starts_nothing(tmp_path):
delivered_case(tmp_path)
decider = FakeDecider({"intake": Answer("change_request", 0.98)})

View File

@@ -62,3 +62,100 @@ async def test_a_change_needs_something_delivered_to_change(tmp_path):
assert "- Design thing 1." in decider.asked[0][0]
assert "Design thing 2." not in decider.asked[0][0] # still in progress
assert without.kind == NEW_REQUEST and no_case.kind == NEW_REQUEST
# --- a question about what was delivered ------------------------------------------------
async def test_a_question_about_delivered_work_is_told_from_one_about_the_service(
tmp_path,
):
delivered(tmp_path, "acme", ["completed"])
about = FakeDecider(
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.9, 0.9)}
)
general = FakeDecider(
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.1, 0.9)}
)
first = await Intake(about, tmp_path).read("Why Cloud SQL?", "acme")
second = await Intake(general, tmp_path).read("What can you do?", "acme")
assert first.kind == QUESTION and first.about_delivered and first.latest == "v1"
assert second.kind == QUESTION and not second.about_delivered
async def test_with_nothing_delivered_a_question_is_never_about_delivered_work(
tmp_path,
):
decider = FakeDecider(
{"intake": Answer(QUESTION, 0.98), "about_delivered": Answer(0.99, 0.9)}
)
reading = await Intake(decider, tmp_path).read("Why Cloud SQL?", "new-client")
assert not reading.about_delivered
assert len(decider.asked) == 1 # the second question was not asked
async def test_an_answer_is_written_from_the_delivered_guide_and_nothing_else(
tmp_path,
):
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission
from app.adk.runtime import RoleSpec
from app.agent import ExecutionEngine
from app.nodes.intake import DeliveredAnswers
from app.nodes.router import Router, Thresholds
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH
roles = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
seen: dict = {}
def writer(task):
seen.update(task.context)
return [Call("write_reply", {"text": "Cloud SQL, for relational bookings."})]
runtime = FakeAgentRuntime(
{
**{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
},
"reply-writer": [writer],
}
)
engine = ExecutionEngine(
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
await engine.start(
"Design bookings.", customer_case="acme", pre_approved=True, execution_id="v1"
)
runs_before = len(runtime.runs)
answers = DeliveredAnswers(
runtime, RoleSpec("reply-writer", "Write.", budget=3), tmp_path
)
answer = await answers.answer("Why Cloud SQL?", "acme", "v1")
assert answer == "Cloud SQL, for relational bookings."
assert seen["question"] == "Why Cloud SQL?"
assert seen["delivered"]["execution_id"] == "v1"
assert seen["delivered"]["request"] == "Design bookings."
assert seen["delivered"]["solution_guide"]
assert len(runtime.runs) == runs_before + 1 # one writer run; no Execution
# An Execution that is not there gives no answer, not an error.
assert await answers.answer("Why?", "acme", "missing") is None

View File

@@ -0,0 +1,100 @@
"""An Execution's files are on disk as soon as each Deliverable is submitted.
Before this they were written only when an Execution ended, so a process that was
killed, or an Execution left waiting for its user, had nothing to show.
"""
import json
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import FakeAgentRuntime, sample_submission
from app.adk.runtime import NoDecider, RoleSpec
from app.agent import ExecutionEngine
from app.baseline import BaselineError, load_baseline
from app.nodes.execution_nodes import _write_outputs
from app.nodes.router import Router, Thresholds
from app.recorder import InMemoryRecorder
from app.states.ledger import Ledger
from app.workflows.deliverables import DEPENDENCY_GRAPH
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def folder(tmp_path, execution_id: str):
return tmp_path / "deliverables" / "cases" / "acme" / "executions" / execution_id
async def test_an_execution_waiting_for_its_user_has_written_what_exists(tmp_path):
engine = ExecutionEngine(
runtime=FakeAgentRuntime(
{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
),
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
)
waiting = await engine.start("Ingest", customer_case="acme", execution_id="v1")
# It stopped for the first Approval: only the first Phase has been produced.
assert waiting.status == "awaiting_user"
out = folder(tmp_path, "v1")
manifest = json.loads((out / "execution.json").read_text())
assert manifest["status"] == "in_progress"
assert set(manifest["deliverables"]) == {"requirements", "technical_decomposition"}
assert (out / "deliverables" / "requirements.md").read_text()
assert (out / "requirements.json").is_file()
assert manifest["solution_guide"] is None
# Unfinished work is never something a Change request can start from.
try:
load_baseline(tmp_path, "acme", "v1")
except BaselineError as refused:
assert "did not complete (in_progress)" in str(refused)
else:
raise AssertionError("an in-progress Execution was accepted as a Baseline")
def test_a_file_an_earlier_write_left_is_removed_with_the_revision(tmp_path):
state = {
"execution_id": "v1",
"customer_case": "acme",
"workflow_request": "Ingest",
"deliverables": {
"deployment_guidance": {
"content": "guide",
"attachments": {
"terraform/main.tf": "one",
"terraform/queue.tf": "two",
},
}
},
}
out = folder(tmp_path, "v1")
_write_outputs(tmp_path, state, "in_progress", Ledger())
assert (out / "terraform" / "queue.tf").is_file()
(out / "notes.txt").write_text("a person's own file")
# The Revision folded the queue into main.tf.
state["deliverables"]["deployment_guidance"]["attachments"] = {
"terraform/main.tf": "one and two"
}
_write_outputs(tmp_path, state, "completed", Ledger())
assert not (out / "terraform" / "queue.tf").exists()
assert (out / "terraform" / "main.tf").read_text() == "one and two"
# Only files this Execution wrote are ever removed.
assert (out / "notes.txt").is_file()
assert json.loads((out / "execution.json").read_text())["status"] == "completed"

View File

@@ -498,16 +498,68 @@ def test_lessons_that_cost_quality_or_save_nothing_are_not_shown_to_help(tmp_pat
assert "nothing fell" in render(compare(base, same), base, same)
def test_the_comparison_is_written_into_published_proposals_only(tmp_path):
def test_the_comparison_is_written_only_for_lessons_in_the_skill(tmp_path):
from app.learning.proof import record
skill = lessons_skill(tmp_path)
issues = tmp_path / "issues"
live = publish(approve(proposal(issues), issues), skill, today=DAY)
waiting = proposal(issues, title="Another", evidence_key="zzz")
gone = publish(
approve(proposal(issues, title="Retired", evidence_key="yyy"), issues),
skill,
today=DAY,
)
retire(gone.id, skill)
recorded = record([live, waiting], "| a table |")
recorded = record([live, waiting, gone], "| a table |", published(skill))
assert [p.id for p in recorded] == ["L-01"]
assert parse(live.path).proof == "| a table |"
assert parse(waiting.path).proof == ""
# A retired lesson was not part of what was compared.
assert parse(gone.path).proof == ""
# --- lessons are read again when the Skills beside them change ---------------------------
def test_a_changed_upstream_skill_is_flagged_until_the_lessons_are_reviewed(tmp_path):
from app.learning.skill import mark_reviewed, upstream_changes
skill = lessons_skill(tmp_path)
issues = tmp_path / "issues"
publish(approve(proposal(issues), issues), skill, today=DAY)
then = {"architecture": "aaa", "security": "bbb"}
mark_reviewed(skill, then)
assert upstream_changes(skill, then) == []
now = {"architecture": "aaa", "security": "ccc", "cost": "ddd"}
assert upstream_changes(skill, now) == ["cost", "security"]
mark_reviewed(skill, now)
assert upstream_changes(skill, now) == []
def test_nothing_is_flagged_with_no_lessons_or_nothing_recorded(tmp_path):
from app.learning.skill import mark_reviewed, upstream_changes
skill = lessons_skill(tmp_path)
assert upstream_changes(skill, {"architecture": "aaa"}) == []
mark_reviewed(skill, {"architecture": "aaa"})
# No lesson is published, so there is nothing to review.
assert upstream_changes(skill, {"architecture": "zzz"}) == []
def test_the_published_lessons_were_reviewed_against_todays_vendored_skills():
"""Syncing the vendored Skills changes what the lessons sit beside. This fails
until a person has read each lesson again and run `curate_lessons reviewed`."""
from app.learning.skill import upstream_changes, upstream_reviewed
from app.providers import load_provider
from eval.curate_lessons import upstream_versions
bundle = load_provider("gcp")
skill = bundle.skills[bundle.lessons_skill].path
assert upstream_reviewed(skill), "run: python -m eval.curate_lessons reviewed"
assert upstream_changes(skill, upstream_versions(bundle)) == []