feat: overlay Skills, and five more Jev decisions (iterations 4 and 5)

Iteration 4:
- Overlay Skills (step 12): an Execution names an organisation's standards
  Skill from the folders the deployment configures; recorded with its version.
  An example standards Skill and an evaluation case that uses it.
- Lessons (step 10): a prove command that compares a run with lessons against
  one without, and triage of evidence an existing lesson already covers.

Iteration 5:
- Step 8: the references of a Specialist's Skills its task needs are named in
  its context as read_first.
- Step 14: a remote MCP server's reply that instructs the agent is withheld;
  the user's own text is flagged as a risk and nothing more.
- Step 15: in a Change request, items only reworded are listed apart from
  real changes.
- Step 19: each Execution's design recommendations are judged when it ends and
  the scores kept in its record. The evaluation takes its rubric from the same
  place.

Each question was calibrated against live Jev; none has run in a case. Steps
6, 7, 9, 13 and 18 stay planned, with the reasons in the plan map.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 17:45:56 +01:00
parent 844987587c
commit 567aede5ca
38 changed files with 1956 additions and 58 deletions

View File

@@ -6,18 +6,21 @@
status list every proposal and where it stands.
publish write every approved proposal (Status: ready-for-agent) into the
provider's lessons Skill.
prove compare an evaluation run without lessons with one run with them,
and write the comparison into every published proposal.
retire remove a lesson from the Skill, by id.
Usage:
.venv/bin/python -m eval.curate_lessons propose [--from <folder> ...] [--min 2]
.venv/bin/python -m eval.curate_lessons status
.venv/bin/python -m eval.curate_lessons publish
.venv/bin/python -m eval.curate_lessons prove --without <report.json> --with <report.json>
.venv/bin/python -m eval.curate_lessons retire L-03
A person approves a proposal by editing its file: set `Status: ready-for-agent`, or
`wontfix` to reject it. Nothing is published without that. After publishing, prove
the lesson: run the evaluation with LESSONS_ENABLED=false and again with it on, and
record the comparison under the proposal's `## Proof`. Retire a lesson that does
the lessons: run the evaluation with LESSONS_ENABLED=false and again with it on, on
the same cases, then run `prove` with the two reports. Retire a lesson that does
not hold quality and lower Revisions, time or tokens.
`propose` costs model usage. Lesson candidates are client data; they are read
@@ -31,9 +34,11 @@ import sys
from collections.abc import Sequence
from pathlib import Path
from app.agent import open_decider
from app.config import get_settings
from app.learning.curator import LESSON_CURATOR, curate
from app.learning.evidence import group_candidates, identifiers, read_candidates
from app.learning.proof import compare, record, render
from app.learning.proposals import ISSUES, load
from app.learning.skill import NotApproved, publish, published, retire
from app.providers import ProviderBundle, load_provider
@@ -79,14 +84,21 @@ async def propose_lessons(arguments: Sequence[str]) -> int:
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
)
proposed = await curate(
runtime,
roles[LESSON_CURATOR],
groups,
executions=len(records),
identifiers=identifiers(records),
folder=ROOT / ISSUES,
)
decider = open_decider(settings)
try:
proposed = await curate(
runtime,
roles[LESSON_CURATOR],
groups,
executions=len(records),
identifiers=identifiers(records),
folder=ROOT / ISSUES,
decider=decider,
)
finally:
aclose = getattr(decider, "aclose", None)
if aclose:
await aclose()
for proposal in proposed:
print(f" {proposal.id} [{proposal.deliverable}] {proposal.title}")
print(
@@ -127,6 +139,23 @@ def publish_approved() -> int:
return 0
def prove_lessons(arguments: Sequence[str]) -> int:
without, with_lessons = (
_options(arguments, "--without"),
_options(arguments, "--with"),
)
if not without or not with_lessons:
print("Give both reports: --without <report.json> --with <report.json>.")
return 2
first, second = Path(without[0]), Path(with_lessons[0])
result = compare(first, second)
proof = render(result, first, second)
recorded = record(load(ROOT / ISSUES), proof)
print(proof)
print(f"\nRecorded in {len(recorded)} published proposals.")
return 0 if result["helps"] else 1
def retire_lesson(arguments: Sequence[str]) -> int:
if not arguments:
print("Name the lesson to retire, such as L-03.", file=sys.stderr)
@@ -145,6 +174,8 @@ def main(arguments: Sequence[str]) -> int:
return show_status()
if command == "publish":
return publish_approved()
if command == "prove":
return prove_lessons(rest)
if command == "retire":
return retire_lesson(rest)
print(__doc__, file=sys.stderr)

View File

@@ -367,5 +367,80 @@
"Approved. Please proceed."
]
]
},
{
"id": "case-06-organisation-standards",
"name": "Internal booking web application, under an organisation's standards (overlay Skill)",
"workflow_request": "Design an internal web application on Google Cloud for staff to book meeting rooms and equipment, backed by a managed relational database, available only to employees signed in with the corporate identity provider, with application secrets kept out of code and configuration.",
"required_products": [
"Cloud Run",
"Cloud SQL",
"Secret Manager"
],
"scripted_answers": [
[
"region|location|geograph",
"A single region, europe-west1 (Belgium). All users are in the EU."
],
[
"user|staff|employee|how many|concurren|traffic|volume|scale|peak|requests per|rps",
"About 4,000 employees; 300 concurrent users at the 09:00 peak, around 20 requests per second."
],
[
"availability|uptime|sla|rto|rpo|disaster|recover|backup",
"99.5% during business hours (07:00-19:00 CET). RTO 4 hours, RPO 1 hour. Daily backups kept 14 days."
],
[
"latency|response time|performance",
"Pages should load within 1 second at p95."
],
[
"identity|sign.?in|login|sso|auth|idp|access",
"Employees sign in with Google Workspace accounts; no access from outside the company. Admins are a Google group."
],
[
"security|compliance|encrypt|privacy|gdpr|secret|credential|password",
"GDPR applies to employee names and emails. Google-managed keys are fine. Database passwords and API keys must live in a secrets store and rotate every 90 days."
],
[
"data|database|schema|relational|postgres|mysql|size|storage",
"PostgreSQL. Under 20 GB of data in the first three years; bookings, rooms, equipment and audit history."
],
[
"cost|budget|spend",
"Under 600 EUR a month; the database can be small."
],
[
"current|existing|today|on-prem|migrat|legacy",
"Bookings are kept in a shared spreadsheet today; there is no data to migrate."
],
[
"depend|integrat|calendar|email|notif|downstream|upstream",
"Send booking confirmation emails through the existing SMTP relay; no calendar integration in phase one."
],
[
"operat|monitor|logging|alert|team|support|deploy|ci|pipeline",
"Two developers own it, deploy containers from Cloud Build with Terraform, and want Cloud Monitoring alerts to a Google Chat space."
],
[
"network|vpc|private|internet|egress|ingress",
"No public database endpoint; the database must use private IP only."
],
[
"sustainab|carbon",
"No specific requirement."
],
[
"retention|audit|archive|delete",
"Keep booking history 2 years, then delete; keep audit logs 1 year."
],
[
"approve|confirm|proceed|look right|any changes|happy",
"Approved. Please proceed."
]
],
"overlay_skills": [
"example-organisation-standards"
]
}
]

View File

@@ -15,6 +15,7 @@ from collections.abc import Mapping
from typing import Any, Protocol
from app.adk.runtime import Decider, Question
from app.tools.assessment import GRADES
INSTRUCTIONS = (
"You grade a document from a cloud solution architecture engagement. Answer every "
@@ -36,9 +37,6 @@ class Judge(Protocol):
...
GRADES = ("not at all", "weakly", "mostly", "fully")
class JudgeUnavailable(RuntimeError):
"""The judge gave no answer; the check it serves is unanswered, not failed."""

View File

@@ -20,6 +20,7 @@ from typing import Any
from app.adk.runtime import ToolResult
from app.agent import Interaction
from app.tools.assessment import QUALITY_BAR, QUALITY_RUBRIC
from app.tools.local import mermaid_check
from app.workflows.chain import (
COMPONENTS_FILE,
@@ -44,20 +45,8 @@ from eval.judge import Judge, JudgeUnavailable
PASS_THRESHOLD = 0.8
CHECKLIST_COVERAGE = 0.7 # share of a pillar Skill's checklist the design must address
QUALITY_BAR = 0.67 # mean of the graded rubric: "mostly" or better
THRESHOLD_METRICS = ("revisions", "seconds", "tokens")
# What a reviewer would ask of design recommendations, graded 0 to 1 each.
QUALITY_RUBRIC: Mapping[str, str] = {
"specific": "Are the recommendations specific to this workload and its stated "
"requirements, as opposed to generic best practice that would fit any workload?",
"actionable": "Does each recommendation name the concrete setting, feature or "
"configuration to apply, so an engineer could act on it without more research?",
"trade_offs": "Where pillars pull in different directions (for example cost "
"against reliability), is the trade-off named and a resolution recommended?",
"traceable": "Can each recommendation be traced to a requirement or constraint "
"of the workload?",
}
TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]]
_HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE)
@@ -80,6 +69,8 @@ class CaseSpec:
# A change case: the request is run to completion, then this statement of what
# has changed is run as a Change request against it (ADR-0006).
change: str = ""
# Overlay Skills the case's Executions name (ADR-0004).
overlay_skills: tuple[str, ...] = ()
@dataclass(frozen=True)
@@ -122,6 +113,7 @@ def load_cases(path: Path) -> list[CaseSpec]:
interactive=bool(item.get("interactive", False)),
thresholds=dict(item.get("thresholds", {})),
change=str(item.get("change", "")),
overlay_skills=tuple(item.get("overlay_skills", ())),
)
for item in json.loads(path.read_text(encoding="utf-8"))
]

View File

@@ -31,7 +31,7 @@ from pathlib import Path
from typing import Any
from app.adk.runtime import AgentRuntime, AgentTask, Outcome, RoleSpec, ToolSpec
from app.agent import ExecutionEngine, Interaction, open_assembly
from app.agent import ExecutionEngine, Interaction, open_assembly, skill_dirs
from app.config import Settings, get_settings
from app.nodes.execution_nodes import automatic_reply
from app.recorder import InMemoryRecorder
@@ -131,6 +131,7 @@ async def run_with_scripted_user(
customer_case=case.id,
allow_cloud_read=False,
execution_id=execution_id,
overlay_skills=case.overlay_skills,
)
for _ in range(max_user_turns):
if interaction.status != "awaiting_user" or interaction.prompt is None:
@@ -150,7 +151,7 @@ async def run_case(
bundle = load_provider(settings.CLOUD_PROVIDER)
adk_runtime = AdkAgentRuntime(
skill_dirs={name: meta.path for name, meta in bundle.skills.items()},
skill_dirs=skill_dirs(settings, bundle),
model_timeout_seconds=settings.MODEL_TIMEOUT_SECONDS,
)
started = time.monotonic()
@@ -192,6 +193,7 @@ async def run_case(
allow_cloud_read=False,
execution_id=f"{execution_id}-base",
scripted_answers=case.scripted_answers,
overlay_skills=case.overlay_skills,
)
baseline_tokens = token_report(adk_runtime.usage)["total"].get("total")
baseline_seconds = round(time.monotonic() - started, 1)
@@ -205,6 +207,7 @@ async def run_case(
allow_cloud_read=False,
execution_id=execution_id,
scripted_answers=case.scripted_answers,
overlay_skills=case.overlay_skills,
baseline=base.execution_id,
)
elif case.interactive:
@@ -219,6 +222,7 @@ async def run_case(
allow_cloud_read=False,
execution_id=execution_id,
scripted_answers=case.scripted_answers,
overlay_skills=case.overlay_skills,
)
elapsed = time.monotonic() - started
write_progress()