chore: housekeeping from the runs of 2026-10-09
Some checks failed
validation / verify (push) Failing after 12s

- An Execution's files are written as each Deliverable is submitted, marked
  in_progress, so a killed process or one waiting for its user leaves them on
  disk. A write removes files an earlier write left that it no longer has.
- A question about delivered work is answered from that Execution's solution
  guide by the reply writer; a question about the service is answered as
  before. Nothing is started for either.
- Lessons: prove writes only for lessons the Skill holds; the Skill records
  the vendored Skill versions the lessons were read against, status names any
  that changed, and a test fails until they are reviewed.
- Four more code patterns: Firestore, BigQuery, Eventarc and GKE.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-10 11:44:38 +01:00
parent 10d23a5f91
commit 71da0ed504
24 changed files with 1109 additions and 43 deletions

View File

@@ -9,6 +9,8 @@
prove compare an evaluation run without lessons with one run with them,
and write the comparison into every published proposal.
retire remove a lesson from the Skill, by id.
reviewed record that the lessons were read against the vendored Skills as
they are now. `status` says when those have changed since.
Usage:
.venv/bin/python -m eval.curate_lessons propose [--from <folder> ...] [--min 2]
@@ -16,6 +18,7 @@ Usage:
.venv/bin/python -m eval.curate_lessons publish
.venv/bin/python -m eval.curate_lessons prove --without <report.json> --with <report.json>
.venv/bin/python -m eval.curate_lessons retire L-03
.venv/bin/python -m eval.curate_lessons reviewed
A person approves a proposal by editing its file: set `Status: ready-for-agent`, or
`wontfix` to reject it. Nothing is published without that. After publishing, prove
@@ -40,8 +43,17 @@ from app.learning.curator import LESSON_CURATOR, curate
from app.learning.evidence import group_candidates, identifiers, read_candidates
from app.learning.proof import compare, record, render
from app.learning.proposals import ISSUES, load
from app.learning.skill import NotApproved, publish, published, retire
from app.learning.skill import (
NotApproved,
mark_reviewed,
publish,
published,
retire,
upstream_changes,
upstream_reviewed,
)
from app.providers import ProviderBundle, load_provider
from app.skills.agent_skills import skill_version
ROOT = Path(__file__).resolve().parent.parent
# Where Executions write their candidates: under each evaluation run, and under the
@@ -55,6 +67,16 @@ def _skill_dir(bundle: ProviderBundle) -> Path:
return bundle.skills[bundle.lessons_skill].path
def upstream_versions(bundle: ProviderBundle) -> dict[str, str]:
"""The vendored Skills, each with the version it has now. They are what the
lessons correct the use of, and they change when upstream is synced."""
return {
name: skill_version(meta.path)
for name, meta in bundle.skills.items()
if "vendor" in meta.path.parts
}
def _options(arguments: Sequence[str], name: str) -> list[str]:
return [
arguments[index + 1]
@@ -115,12 +137,34 @@ def show_status() -> int:
if not proposals:
print("No lesson has been proposed.")
for proposal in proposals:
where = "in the Skill" if proposal.id in live else "not in the Skill"
proof = "proved" if proposal.proof else "not proved"
where = (
"in the Skill"
if proposal.id in live
else "retired"
if proposal.published
else "not in the Skill"
)
# One comparison suggests a direction; it does not prove a lesson.
proof = "compared" if proposal.proof else "not compared"
print(
f"{proposal.id} {proposal.status:16s} {where:16s} {proof:11s} "
f"{proposal.id} {proposal.status:16s} {where:16s} {proof:12s} "
f"[{proposal.deliverable}] {proposal.title}"
)
changed = upstream_changes(_skill_dir(bundle), upstream_versions(bundle))
if changed:
print(
f"\nVendored Skills changed since the lessons were last reviewed: "
f"{', '.join(changed)}.\nRead each lesson in the Skill against them: "
"retire one the Skill now says itself or contradicts. Then run "
"`reviewed`."
)
return 0
def mark_lessons_reviewed() -> int:
bundle = load_provider(get_settings().CLOUD_PROVIDER)
mark_reviewed(_skill_dir(bundle), upstream_versions(bundle))
print("Recorded: the lessons stand against the vendored Skills as they are now.")
return 0
@@ -135,6 +179,10 @@ def publish_approved() -> int:
continue
done += 1
print(f" published {proposal.id} [{proposal.deliverable}] {proposal.title}")
if done and not upstream_reviewed(skill_dir):
# The first lessons: what they were written beside is recorded once. After
# that only a person's review moves it on.
mark_reviewed(skill_dir, upstream_versions(bundle))
print(f"{done} lessons are in {skill_dir.name}.")
return 0
@@ -150,7 +198,8 @@ def prove_lessons(arguments: Sequence[str]) -> int:
first, second = Path(without[0]), Path(with_lessons[0])
result = compare(first, second)
proof = render(result, first, second)
recorded = record(load(ROOT / ISSUES), proof)
bundle = load_provider(get_settings().CLOUD_PROVIDER)
recorded = record(load(ROOT / ISSUES), proof, published(_skill_dir(bundle)))
print(proof)
print(f"\nRecorded in {len(recorded)} published proposals.")
return 0 if result["helps"] else 1
@@ -178,6 +227,8 @@ def main(arguments: Sequence[str]) -> int:
return prove_lessons(rest)
if command == "retire":
return retire_lesson(rest)
if command == "reviewed":
return mark_lessons_reviewed()
print(__doc__, file=sys.stderr)
return 2