feat: overlay Skills, and five more Jev decisions (iterations 4 and 5)
Iteration 4: - Overlay Skills (step 12): an Execution names an organisation's standards Skill from the folders the deployment configures; recorded with its version. An example standards Skill and an evaluation case that uses it. - Lessons (step 10): a prove command that compares a run with lessons against one without, and triage of evidence an existing lesson already covers. Iteration 5: - Step 8: the references of a Specialist's Skills its task needs are named in its context as read_first. - Step 14: a remote MCP server's reply that instructs the agent is withheld; the user's own text is flagged as a risk and nothing more. - Step 15: in a Change request, items only reworded are listed apart from real changes. - Step 19: each Execution's design recommendations are judged when it ends and the scores kept in its record. The evaluation takes its rubric from the same place. Each question was calibrated against live Jev; none has run in a case. Steps 6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -399,3 +399,115 @@ def test_every_published_lesson_has_an_approved_proposal():
|
||||
for lesson, deliverable in live.items():
|
||||
assert approved[lesson].deliverable == deliverable
|
||||
assert approved[lesson].published
|
||||
|
||||
|
||||
# --- triage: evidence a lesson already covers ---------------------------------------------
|
||||
|
||||
|
||||
async def test_a_problem_an_existing_lesson_covers_is_not_put_to_the_curator(tmp_path):
|
||||
from app.adk.fake_runtime import FakeDecider
|
||||
from app.adk.runtime import Answer
|
||||
|
||||
proposal(tmp_path, evidence_key="older-evidence") # L-01 exists
|
||||
groups = group_candidates(RECORDS)
|
||||
decider = FakeDecider({groups[0].key: Answer("L-01", 0.9, {"L-01": 0.9})})
|
||||
runtime = FakeAgentRuntime({})
|
||||
|
||||
proposed = await curate(
|
||||
runtime, CURATOR, groups, executions=3, folder=tmp_path, decider=decider
|
||||
)
|
||||
|
||||
assert proposed == [] and runtime.runs == []
|
||||
state, questions, _ = decider.asked[0]
|
||||
assert "- L-01: Before you submit, give every Element" in state
|
||||
assert questions[groups[0].key].options == ("L-01", "none")
|
||||
|
||||
|
||||
async def test_a_problem_no_lesson_covers_still_reaches_the_curator(tmp_path):
|
||||
from app.adk.fake_runtime import FakeDecider
|
||||
from app.adk.runtime import Answer
|
||||
|
||||
proposal(tmp_path, evidence_key="older-evidence")
|
||||
groups = group_candidates(RECORDS)
|
||||
decider = FakeDecider({groups[0].key: Answer("none", 0.95, {"none": 0.95})})
|
||||
runtime = curator_calls(
|
||||
Call("no_lesson", {"group": groups[0].key, "why": "The engine refuses it."})
|
||||
)
|
||||
|
||||
await curate(
|
||||
runtime, CURATOR, groups, executions=3, folder=tmp_path, decider=decider
|
||||
)
|
||||
|
||||
assert len(runtime.runs) == 1
|
||||
|
||||
|
||||
# --- prove: with against without -----------------------------------------------------------
|
||||
|
||||
|
||||
def report(tmp_path: Path, name: str, **cases) -> Path:
|
||||
folder = tmp_path / name
|
||||
folder.mkdir()
|
||||
path = folder / "report.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"case_id": case,
|
||||
"score": score,
|
||||
"details": {
|
||||
"revisions": {"target_architecture": revisions},
|
||||
"seconds": minutes * 60,
|
||||
"tokens": {"total": {"total": tokens}},
|
||||
},
|
||||
}
|
||||
for case, (score, revisions, minutes, tokens) in cases.items()
|
||||
]
|
||||
)
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def test_lessons_help_when_quality_holds_and_something_falls(tmp_path):
|
||||
from app.learning.proof import compare, render
|
||||
|
||||
without = report(
|
||||
tmp_path, "run-a", case_02=(1.0, 2, 40, 5_000_000), other=(1, 0, 1, 1)
|
||||
)
|
||||
with_lessons = report(tmp_path, "run-b", case_02=(1.0, 0, 25, 3_000_000))
|
||||
|
||||
result = compare(without, with_lessons)
|
||||
|
||||
assert result["helps"] and result["lower"] == ["revisions", "seconds", "tokens"]
|
||||
assert list(result["cases"]) == ["case_02"] # only what both ran
|
||||
text = render(result, without, with_lessons)
|
||||
assert "| case_02 | 1.00 → 1.00 | 2 → 0 | 40 → 25 | 5.00 → 3.00 |" in text
|
||||
assert "Quality held and revisions, seconds, tokens fell: kept." in text
|
||||
assert "`run-a`" in text and "`run-b`" in text
|
||||
|
||||
|
||||
def test_lessons_that_cost_quality_or_save_nothing_are_not_shown_to_help(tmp_path):
|
||||
from app.learning.proof import compare, render
|
||||
|
||||
base = report(tmp_path, "a", case_02=(1.0, 1, 30, 4_000_000))
|
||||
worse = report(tmp_path, "b", case_02=(0.9, 0, 20, 3_000_000))
|
||||
same = report(tmp_path, "c", case_02=(1.0, 1, 31, 4_100_000))
|
||||
|
||||
assert not compare(base, worse)["helps"]
|
||||
assert "scored lower" in render(compare(base, worse), base, worse)
|
||||
assert not compare(base, same)["helps"]
|
||||
assert "nothing fell" in render(compare(base, same), base, same)
|
||||
|
||||
|
||||
def test_the_comparison_is_written_into_published_proposals_only(tmp_path):
|
||||
from app.learning.proof import record
|
||||
|
||||
skill = lessons_skill(tmp_path)
|
||||
issues = tmp_path / "issues"
|
||||
live = publish(approve(proposal(issues), issues), skill, today=DAY)
|
||||
waiting = proposal(issues, title="Another", evidence_key="zzz")
|
||||
|
||||
recorded = record([live, waiting], "| a table |")
|
||||
|
||||
assert [p.id for p in recorded] == ["L-01"]
|
||||
assert parse(live.path).proof == "| a table |"
|
||||
assert parse(waiting.path).proof == ""
|
||||
|
||||
Reference in New Issue
Block a user