Design is now approved before any infrastructure code is written (ADR-0005). A new target-architect Specialist produces one graph of Elements, Relationships and Checks; the diagram is rendered from it, deployment guidance must account for every Check, and validation gives a verdict per Check. Requirements, components, products and recommendations carry ids in data files, and a submission with a gap in the Requirements chain is refused. Also in this commit: - Customer cases group Executions, with an intent history per case - lesson candidates are captured when an Execution ends - evaluation gains chain, verdict, judged and interactive checks, and per-case thresholds - Terraform providers are cached between checks - the requirements analyst proposes defaults instead of asking - the legacy-baseline case and the checks that only read it are removed - ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed - plans for iterations 4 to 7 under .agents/plans/ Unproven with a real model: no evaluation run has completed on this design. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
126 lines
4.1 KiB
Python
126 lines
4.1 KiB
Python
"""Lesson candidates: what went wrong in an Execution and what was asked to fix it.
|
|
|
|
Collected once, when the Execution ends, from what the Execution already holds: the
|
|
Ledger's events, the Specialists' outcome records and the tool refusals. They are raw
|
|
material for a reviewed change to a Skill, never memory carried into another Execution.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Mapping, Sequence
|
|
from typing import Any
|
|
|
|
from app.states.ledger import Ledger
|
|
from app.workflows.chain import VERDICTS_FILE, validation_outcome
|
|
from app.workflows.deliverables import (
|
|
DEPENDENCY_GRAPH,
|
|
DeliverableKind,
|
|
PhaseId,
|
|
)
|
|
|
|
MESSAGE_LIMIT = 500
|
|
_UNFINISHED = {"budget_exhausted", "stopped"}
|
|
|
|
|
|
def refusal(
|
|
role: str, deliverable: str | None, tool: str, tier: str, message: str
|
|
) -> dict[str, Any]:
|
|
"""One tool refusal, as stored in Execution state.
|
|
|
|
A cloud-read tool's message may describe the user's environment, so only its
|
|
leading words are kept.
|
|
"""
|
|
if tier == "cloud_read":
|
|
message = message.split(":", 1)[0]
|
|
return {
|
|
"role": role,
|
|
"deliverable": deliverable,
|
|
"tool": tool,
|
|
"message": message[:MESSAGE_LIMIT],
|
|
}
|
|
|
|
|
|
def collect_lesson_candidates(
|
|
*,
|
|
execution_id: str,
|
|
customer_case: str,
|
|
status: str,
|
|
ledger: Ledger,
|
|
outcomes: Sequence[Mapping[str, Any]],
|
|
orchestrator_refusals: Sequence[Mapping[str, Any]],
|
|
role_skills: Mapping[str, Sequence[str]],
|
|
skill_versions: Mapping[str, str],
|
|
) -> dict[str, Any]:
|
|
candidates: list[dict[str, Any]] = []
|
|
|
|
for event in ledger.events:
|
|
if event.type == "reopened":
|
|
candidates.append(
|
|
{
|
|
"type": "revision",
|
|
"deliverable": event.subject,
|
|
"reason": event.detail,
|
|
"requested_by": event.actor,
|
|
}
|
|
)
|
|
elif event.type == "escalated":
|
|
candidates.append(
|
|
{
|
|
"type": "revision_limit",
|
|
"deliverable": event.subject,
|
|
"reason": event.detail,
|
|
}
|
|
)
|
|
elif event.type == "changes_requested":
|
|
candidates.append(
|
|
{
|
|
"type": "change_request",
|
|
"phase": event.subject,
|
|
"deliverables": [
|
|
kind.value
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
if spec.phase is PhaseId(event.subject)
|
|
],
|
|
"feedback": event.detail,
|
|
}
|
|
)
|
|
elif event.type == "released" and event.detail in _UNFINISHED:
|
|
candidates.append(
|
|
{
|
|
"type": "unfinished",
|
|
"deliverable": event.subject,
|
|
"role": event.actor,
|
|
"reason": event.detail,
|
|
}
|
|
)
|
|
|
|
for record in outcomes:
|
|
for item in record.get("refusals", ()):
|
|
candidates.append({"type": "refusal", **item})
|
|
if (
|
|
record.get("type") == "submitted"
|
|
and record.get("kind") == DeliverableKind.VALIDATION_RESULTS.value
|
|
):
|
|
outcome = validation_outcome(
|
|
record.get("attachments", {}).get(VERDICTS_FILE, "")
|
|
)
|
|
attempt = record.get("attempt")
|
|
for kind, items in (
|
|
("failed_check", outcome["failed"]),
|
|
("unverifiable_check", outcome["not_verifiable"]),
|
|
("proposed_addition", outcome["proposed_additions"]),
|
|
):
|
|
for item in items:
|
|
candidates.append({"type": kind, "attempt": attempt, **item})
|
|
for item in orchestrator_refusals:
|
|
candidates.append({"type": "refusal", **item})
|
|
|
|
return {
|
|
"execution_id": execution_id,
|
|
"customer_case": customer_case,
|
|
"status": status,
|
|
"role_skills": {role: list(skills) for role, skills in role_skills.items()},
|
|
"skill_versions": dict(skill_versions),
|
|
"candidates": candidates,
|
|
}
|