chore: housekeeping from the runs of 2026-10-09
Some checks failed
validation / verify (push) Failing after 12s
Some checks failed
validation / verify (push) Failing after 12s
- An Execution's files are written as each Deliverable is submitted, marked in_progress, so a killed process or one waiting for its user leaves them on disk. A write removes files an earlier write left that it no longer has. - A question about delivered work is answered from that Execution's solution guide by the reply writer; a question about the service is answered as before. Nothing is started for either. - Lessons: prove writes only for lessons the Skill holds; the Skill records the vendored Skill versions the lessons were read against, status names any that changed, and a test fails until they are reviewed. - Four more code patterns: Firestore, BigQuery, Eventarc and GKE. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,8 @@
|
||||
prove compare an evaluation run without lessons with one run with them,
|
||||
and write the comparison into every published proposal.
|
||||
retire remove a lesson from the Skill, by id.
|
||||
reviewed record that the lessons were read against the vendored Skills as
|
||||
they are now. `status` says when those have changed since.
|
||||
|
||||
Usage:
|
||||
.venv/bin/python -m eval.curate_lessons propose [--from <folder> ...] [--min 2]
|
||||
@@ -16,6 +18,7 @@ Usage:
|
||||
.venv/bin/python -m eval.curate_lessons publish
|
||||
.venv/bin/python -m eval.curate_lessons prove --without <report.json> --with <report.json>
|
||||
.venv/bin/python -m eval.curate_lessons retire L-03
|
||||
.venv/bin/python -m eval.curate_lessons reviewed
|
||||
|
||||
A person approves a proposal by editing its file: set `Status: ready-for-agent`, or
|
||||
`wontfix` to reject it. Nothing is published without that. After publishing, prove
|
||||
@@ -40,8 +43,17 @@ from app.learning.curator import LESSON_CURATOR, curate
|
||||
from app.learning.evidence import group_candidates, identifiers, read_candidates
|
||||
from app.learning.proof import compare, record, render
|
||||
from app.learning.proposals import ISSUES, load
|
||||
from app.learning.skill import NotApproved, publish, published, retire
|
||||
from app.learning.skill import (
|
||||
NotApproved,
|
||||
mark_reviewed,
|
||||
publish,
|
||||
published,
|
||||
retire,
|
||||
upstream_changes,
|
||||
upstream_reviewed,
|
||||
)
|
||||
from app.providers import ProviderBundle, load_provider
|
||||
from app.skills.agent_skills import skill_version
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
# Where Executions write their candidates: under each evaluation run, and under the
|
||||
@@ -55,6 +67,16 @@ def _skill_dir(bundle: ProviderBundle) -> Path:
|
||||
return bundle.skills[bundle.lessons_skill].path
|
||||
|
||||
|
||||
def upstream_versions(bundle: ProviderBundle) -> dict[str, str]:
|
||||
"""The vendored Skills, each with the version it has now. They are what the
|
||||
lessons correct the use of, and they change when upstream is synced."""
|
||||
return {
|
||||
name: skill_version(meta.path)
|
||||
for name, meta in bundle.skills.items()
|
||||
if "vendor" in meta.path.parts
|
||||
}
|
||||
|
||||
|
||||
def _options(arguments: Sequence[str], name: str) -> list[str]:
|
||||
return [
|
||||
arguments[index + 1]
|
||||
@@ -115,12 +137,34 @@ def show_status() -> int:
|
||||
if not proposals:
|
||||
print("No lesson has been proposed.")
|
||||
for proposal in proposals:
|
||||
where = "in the Skill" if proposal.id in live else "not in the Skill"
|
||||
proof = "proved" if proposal.proof else "not proved"
|
||||
where = (
|
||||
"in the Skill"
|
||||
if proposal.id in live
|
||||
else "retired"
|
||||
if proposal.published
|
||||
else "not in the Skill"
|
||||
)
|
||||
# One comparison suggests a direction; it does not prove a lesson.
|
||||
proof = "compared" if proposal.proof else "not compared"
|
||||
print(
|
||||
f"{proposal.id} {proposal.status:16s} {where:16s} {proof:11s} "
|
||||
f"{proposal.id} {proposal.status:16s} {where:16s} {proof:12s} "
|
||||
f"[{proposal.deliverable}] {proposal.title}"
|
||||
)
|
||||
changed = upstream_changes(_skill_dir(bundle), upstream_versions(bundle))
|
||||
if changed:
|
||||
print(
|
||||
f"\nVendored Skills changed since the lessons were last reviewed: "
|
||||
f"{', '.join(changed)}.\nRead each lesson in the Skill against them: "
|
||||
"retire one the Skill now says itself or contradicts. Then run "
|
||||
"`reviewed`."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def mark_lessons_reviewed() -> int:
|
||||
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
||||
mark_reviewed(_skill_dir(bundle), upstream_versions(bundle))
|
||||
print("Recorded: the lessons stand against the vendored Skills as they are now.")
|
||||
return 0
|
||||
|
||||
|
||||
@@ -135,6 +179,10 @@ def publish_approved() -> int:
|
||||
continue
|
||||
done += 1
|
||||
print(f" published {proposal.id} [{proposal.deliverable}] {proposal.title}")
|
||||
if done and not upstream_reviewed(skill_dir):
|
||||
# The first lessons: what they were written beside is recorded once. After
|
||||
# that only a person's review moves it on.
|
||||
mark_reviewed(skill_dir, upstream_versions(bundle))
|
||||
print(f"{done} lessons are in {skill_dir.name}.")
|
||||
return 0
|
||||
|
||||
@@ -150,7 +198,8 @@ def prove_lessons(arguments: Sequence[str]) -> int:
|
||||
first, second = Path(without[0]), Path(with_lessons[0])
|
||||
result = compare(first, second)
|
||||
proof = render(result, first, second)
|
||||
recorded = record(load(ROOT / ISSUES), proof)
|
||||
bundle = load_provider(get_settings().CLOUD_PROVIDER)
|
||||
recorded = record(load(ROOT / ISSUES), proof, published(_skill_dir(bundle)))
|
||||
print(proof)
|
||||
print(f"\nRecorded in {len(recorded)} published proposals.")
|
||||
return 0 if result["helps"] else 1
|
||||
@@ -178,6 +227,8 @@ def main(arguments: Sequence[str]) -> int:
|
||||
return prove_lessons(rest)
|
||||
if command == "retire":
|
||||
return retire_lesson(rest)
|
||||
if command == "reviewed":
|
||||
return mark_lessons_reviewed()
|
||||
print(__doc__, file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
Reference in New Issue
Block a user