diff --git a/.env.example b/.env.example index 8e66a78..1201339 100644 --- a/.env.example +++ b/.env.example @@ -71,6 +71,9 @@ JEV_COVERAGE_THRESHOLD=0.6 # product? Off until a full case has run with it. JEV_REQUIREMENTS_REVIEW=false JEV_REQUIREMENTS_THRESHOLD=0.3 +# How directly a change must affect a pillar for its advisor to be consulted again: +# 0 not at all, 1 only indirectly, 2 directly. +JEV_PILLAR_THRESHOLD=1.5 # PostgreSQL (Execution records, and checkpoints when CHECKPOINTER=postgres) POSTGRES_HOST=localhost diff --git a/agents/design-advisor.md b/agents/design-advisor.md index 58006f0..02cc41d 100644 --- a/agents/design-advisor.md +++ b/agents/design-advisor.md @@ -7,7 +7,8 @@ includes: [specialist-protocol] You produce the **design recommendations** Deliverable: the Skill's task for recommending how to configure each component to meet the functional and non-functional requirements. - Call `consult_pillar_advisors` once, with a brief that restates the workload, the relevant requirements and the selected products. It runs one advisor per architecture-framework pillar in parallel and returns their guidance. -- When `feedback` is not empty you are revising: pass `pillars` naming only the pillars the feedback touches. The guidance you were given before is returned for the others. +- When `pillars_to_consult` is a list, pass exactly that list as `pillars`, even when it is empty: those are the pillars the change affects directly, and the guidance you were given before is returned for the others. +- Otherwise, when `feedback` is not empty or `baseline` is set you are revising: pass `pillars` naming only the pillars the change touches. The guidance you were given before is returned for the others. - Reconcile the pillars' guidance into recommendations for each component. Where pillars conflict, name the trade-off and recommend a resolution tied to the requirements. - When an advisor's guidance is missing, derive that pillar's recommendations from the best-practices references in the solution-architecture Skill. - Give every recommendation an id and attach them as `recommendations.json`, in the format the requirements-chain Skill gives. Write one configurable decision per recommendation, so each can become a Check in the Target architecture. Give each its `conditions`: one entry for each separate thing it asks to be configured, and for each component it must hold on when it names several. Put under `outside_code` anything it asks of the application's own code or of people, which no infrastructure configuration can show. diff --git a/app/agent.py b/app/agent.py index c35b5cb..43c536b 100644 --- a/app/agent.py +++ b/app/agent.py @@ -92,6 +92,8 @@ class ExecutionEngine: orchestrator: OrchestratorPolicy | None = None, submission_review: SubmissionReview | None = None, confirm_drafts: bool = False, + decider: Decider | None = None, + pillar_threshold: float = 1.5, ) -> None: required = { ORCHESTRATOR, @@ -114,6 +116,8 @@ class ExecutionEngine: orchestrator=orchestrator, submission_review=submission_review, confirm_drafts=confirm_drafts, + decider=decider, + pillar_threshold=pillar_threshold, ).compile(checkpointer=checkpointer) async def start( @@ -381,6 +385,8 @@ async def open_assembly( orchestrator=orchestrator, submission_review=Reviews(reviews) if reviews else None, confirm_drafts=settings.CONFIRM_DRAFTS, + decider=decider, + pillar_threshold=settings.JEV_PILLAR_THRESHOLD, ) try: yield Assembly( diff --git a/app/baseline.py b/app/baseline.py index 8fcd296..72f2bf5 100644 --- a/app/baseline.py +++ b/app/baseline.py @@ -17,6 +17,7 @@ from app.states.execution_state import UNASSIGNED_CASE, case_dir, execution_dir MANIFEST_FILENAME = "execution.json" COMPLETED = "completed" +WORKING_FILENAME = "working.json" _EXECUTION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") @@ -86,10 +87,21 @@ def load_baseline( } if not deliverables: raise BaselineError(f"baseline {execution_id} holds no Deliverables") + working_path = folder / WORKING_FILENAME + working: dict[str, Any] = {} + if working_path.is_file(): + try: + loaded = json.loads(working_path.read_text(encoding="utf-8")) + except ValueError: + loaded = {} + if isinstance(loaded, dict): + working = loaded return { "execution_id": execution_id, "customer_case": customer_case, "workflow_request": manifest.get("workflow_request", ""), "baseline": manifest.get("baseline"), # what it was itself changed from "deliverables": deliverables, + # Working material worth starting from again, such as pillar advice. + "working": working, } diff --git a/app/config.py b/app/config.py index 89b45c8..c129673 100644 --- a/app/config.py +++ b/app/config.py @@ -97,6 +97,9 @@ class Settings(BaseSettings): # until a full case has run with it. JEV_REQUIREMENTS_REVIEW: bool = False JEV_REQUIREMENTS_THRESHOLD: float = 0.3 + # How directly a change must affect a pillar for its advisor to be consulted + # again: 0 not at all, 1 only indirectly, 2 directly. + JEV_PILLAR_THRESHOLD: float = 1.5 # Paths BASE_DIR: Path = Path(__file__).resolve().parent.parent diff --git a/app/nodes/execution_nodes.py b/app/nodes/execution_nodes.py index caf2c6b..d4ae9cf 100644 --- a/app/nodes/execution_nodes.py +++ b/app/nodes/execution_nodes.py @@ -23,6 +23,7 @@ from app.adk.runtime import ( AgentRuntime, AgentTask, BudgetExhausted, + Decider, NeedsInput, Outcome, RoleSpec, @@ -34,6 +35,7 @@ from app.adk.runtime import ( ToolSpec, TurnEnded, ) +from app.baseline import WORKING_FILENAME from app.customer_case import close_intent from app.nodes.router import ModelOrchestrator, OrchestratorPolicy, TurnView from app.recorder import ExecutionRecorder @@ -47,6 +49,7 @@ from app.states.ledger import DeliverableStatus, Ledger, LedgerRefusal from app.states.lessons import collect_lesson_candidates, refusal from app.tools import workspace from app.tools.execution import ORCHESTRATOR, OrchestratorToolbox, specialist_tools +from app.tools.pillars import ADVICE_KEY, KEPT_FOR_CHANGES, pillars_touched from app.workflows.chain import ( IMPLEMENTATION_FILE, VERDICTS_FILE, @@ -108,6 +111,8 @@ def make_execution_nodes( orchestrator: OrchestratorPolicy | None = None, submission_review: SubmissionReview | None = None, confirm_drafts: bool = False, + decider: Decider | None = None, + pillar_threshold: float = 1.5, ) -> ExecutionNodes: tool_factory = tool_factory or _no_tools # How an Orchestrator turn is decided. The default drives the tools from the @@ -315,11 +320,14 @@ def make_execution_nodes( ) if isinstance(outcome, TurnEnded) and outcome.tool == "delegate": + hints = await _pillar_hints( + state, ledger, deliverables, toolbox.started, decider, pillar_threshold + ) sends = [ Send( "specialist_task", _specialist_input( - state, ledger, deliverables, drafts, kind, attempt + state, ledger, deliverables, drafts, kind, attempt, hints ), ) for kind, attempt in toolbox.started.items() @@ -391,6 +399,8 @@ def make_execution_nodes( # A Change request starts from the Baseline's version, not from nothing: # its graph, its files and its description are the working copy. _start_from(kind, before, working, task_input.get("baseline_basis")) + # And from its working material: the advice it was written from. + working.update(task_input.get("baseline_working") or {}) source = DEPENDENCY_GRAPH[kind].working_copy if source and source is not kind: # The tools work on the files as they were last submitted. @@ -714,6 +724,7 @@ def _specialist_input( drafts: Mapping[str, Mapping[str, Any]], kind: DeliverableKind, attempt: int, + hints: Mapping[str, Any] | None = None, ) -> dict[str, Any]: spec = DEPENDENCY_GRAPH[kind] given = dependencies_for(kind, deliverables) @@ -737,6 +748,9 @@ def _specialist_input( "shown": kind.value in state.get("drafts_shown", []), # The Baseline's version of this Deliverable, in a Change request. "baseline": before, + "baseline_working": dict( + (baseline or {}).get("working", {}).get(kind.value, {}) + ), # What the Baseline's version was written from, to tell whether it still stands. "baseline_basis": spec.written_from((baseline or {}).get("deliverables")) if baseline and spec.written_from @@ -754,6 +768,9 @@ def _specialist_input( state.get("working", {}).get(kind.value, {}) ), "attempt": attempt, + # The pillars a change touches, when that could be judged: the design + # advisor consults those and keeps its earlier advice for the rest. + "pillars_to_consult": (hints or {}).get(kind.value), # Set only in a Change request: what this Execution changes, and from what. "baseline": _baseline_context(kind, baseline, before, deliverables) if baseline @@ -762,6 +779,44 @@ def _specialist_input( } +async def _pillar_hints( + state: ExecutionState, + ledger: Ledger, + deliverables: Mapping[str, Mapping[str, Any]], + started: Mapping[DeliverableKind, int], + decider: Decider | None, + threshold: float, +) -> dict[str, list[str]]: + """Which pillars the design advisor should consult again, when it is being + started on a Revision or a Change request and advice from before exists.""" + kind = DeliverableKind.DESIGN_RECOMMENDATIONS + if decider is None or kind not in started: + return {} + baseline = state.get("baseline") or {} + remembered = state.get("working", {}).get(kind.value, {}).get( + ADVICE_KEY + ) or baseline.get("working", {}).get(kind.value, {}).get(ADVICE_KEY) + if not remembered: + return {} # a first consultation: every pillar + change = "\n".join(ledger.feedback(kind)) + if not change and baseline: + summary = change_summary(baseline.get("deliverables"), deliverables) + change = "\n".join( + f"Requirement {item['id']} {item['change']}: {item['text']} " + f"({item['reason']})" + for item in summary["requirements"] + ) + chosen = await pillars_touched( + decider, + list(remembered), + change, + threshold=threshold, + scope=state["execution_id"], + ) + # None: no decision, so no hint and every pillar. An empty list is an answer. + return {} if chosen is None else {kind.value: chosen} + + # The solution guide is assembled afresh from the revised Deliverables, so the # Baseline's guide, the longest document there is, is not handed back to its author. _NOT_HANDED_BACK = {DeliverableKind.SOLUTION_GUIDE} @@ -964,6 +1019,14 @@ def _write_outputs( # submitted, or back in draft after something it depends on was reopened "status": ledger.status(DeliverableKind(kind)).value, } + kept = { + kind: {key: memory[key] for key in KEPT_FOR_CHANGES if memory.get(key)} + for kind, memory in state.get("working", {}).items() + } + kept = {kind: memory for kind, memory in kept.items() if memory} + if kept: + # Working material a Change request starts from, such as each pillar's advice. + files[WORKING_FILENAME] = json.dumps(kept, indent=2, ensure_ascii=False) baseline = state.get("baseline") or None if baseline: # What this Change request changed against its Baseline, worked out from diff --git a/app/tools/pillars.py b/app/tools/pillars.py index ea1d0d1..e31b784 100644 --- a/app/tools/pillars.py +++ b/app/tools/pillars.py @@ -22,6 +22,8 @@ from typing import Any from app.adk.runtime import ( AgentRuntime, AgentTask, + Decider, + Question, RoleSpec, RunLimiter, Submitted, @@ -34,6 +36,9 @@ NO_GUIDANCE = "No guidance" ADVICE_KEY = "pillar_advice" # working memory: pillar -> the advice last given CHECKLIST_KEY = "pillar_checklists" # working memory: pillar -> its checklist review REVIEW_HEADING = "Architecture framework checklist review" +# What a Change request carries over from its Baseline's working memory. +KEPT_FOR_CHANGES = (ADVICE_KEY, CHECKLIST_KEY) +_CHANGE_LIMIT = 6_000 STATUSES = ("addressed", "not_applicable", "not_addressed") _STATUS_WORDS = { "addressed": "Addressed", @@ -42,6 +47,53 @@ _STATUS_WORDS = { } +AFFECTED = ("not at all", "only indirectly", "directly") + + +async def pillars_touched( + decider: Decider, + pillars: Sequence[str], + change: str, + *, + threshold: float = 1.5, + scope: str = "", +) -> list[str] | None: + """The pillars whose recommendations a change affects directly (ADR-0007). + + One graded question per pillar: not at all, only indirectly, directly. A pillar + at `threshold` or above, on that scale of 0 to 2, is consulted again. An empty + list means the change touches none of them. None means no decision was + available, and every pillar is consulted. + + Asked as a yes or no ("does this change call for revisiting ...?") the answer + was yes for nearly every pillar of every change; measured on 2026-10-09. + """ + if not change.strip() or not pillars: + return None + answers = await decider.decide( + f"A change to a cloud solution architecture:\n{change[:_CHANGE_LIMIT]}", + { + pillar: Question( + "score", + f"How directly does this change affect the {pillar} " + "recommendations for the workload?", + AFFECTED, + ) + for pillar in pillars + }, + scope=scope, + ) + if not answers: + return None + return [ + pillar + for pillar in pillars + if (answer := answers.get(pillar)) is not None + and isinstance(answer.value, (int, float)) + and answer.value >= threshold + ] + + def pillar_consultation_tool( runtime: AgentRuntime, advisor: RoleSpec, @@ -65,7 +117,9 @@ def pillar_consultation_tool( ) async def consult(brief: str, pillars: Sequence[str] | None = None) -> ToolResult: - chosen = list(pillars) if pillars else list(pillar_skills) + # No `pillars` consults them all; an empty list consults none, and returns + # what was said before. + chosen = list(pillar_skills) if pillars is None else list(pillars) unknown = [pillar for pillar in chosen if pillar not in pillar_skills] if unknown: return ToolResult( @@ -111,7 +165,8 @@ def pillar_consultation_tool( "Get guidance from one advisor per architecture-framework pillar, in parallel. " "Give a brief with the workload, its requirements and the selected products. " "Omit `pillars` to consult every pillar afresh. Name `pillars` to re-consult " - "only those; advice already given for the others is returned unchanged.", + "only those; advice already given for the others is returned unchanged. An " + "empty list consults none and returns the advice already given.", { "type": "object", "properties": { diff --git a/app/workflows/execution_graph.py b/app/workflows/execution_graph.py index f1423bf..6e58b3e 100644 --- a/app/workflows/execution_graph.py +++ b/app/workflows/execution_graph.py @@ -20,6 +20,7 @@ from langgraph.graph import END, START, StateGraph from app.adk.runtime import ( AgentRuntime, + Decider, RoleSpec, RunLimiter, SubmissionReview, @@ -44,6 +45,8 @@ def build_execution_graph( orchestrator: OrchestratorPolicy | None = None, submission_review: SubmissionReview | None = None, confirm_drafts: bool = False, + decider: Decider | None = None, + pillar_threshold: float = 1.5, ) -> StateGraph: nodes = make_execution_nodes( runtime=runtime, @@ -57,6 +60,8 @@ def build_execution_graph( orchestrator=orchestrator, submission_review=submission_review, confirm_drafts=confirm_drafts, + decider=decider, + pillar_threshold=pillar_threshold, ) graph = StateGraph(ExecutionState) diff --git a/tests/execution/test_change_requests.py b/tests/execution/test_change_requests.py index a0647b5..192d257 100644 --- a/tests/execution/test_change_requests.py +++ b/tests/execution/test_change_requests.py @@ -631,3 +631,85 @@ async def test_a_change_request_writes_changes_json_and_a_full_set_of_files(tmp_ assert set(manifest["deliverables"]) == {kind.value for kind in DEPENDENCY_GRAPH} assert (case / "v2" / "solution-architecture-guide.md").is_file() assert not (case / "v1" / "changes.json").exists() + + +# --- only the pillars a change touches are consulted again ------------------------------ + + +async def test_a_change_request_starts_from_the_baselines_pillar_advice(tmp_path): + """The design advisor is told which pillars the change affects, and has the + Baseline's advice for the rest.""" + from app.adk.fake_runtime import FakeDecider + from app.adk.runtime import Answer + from app.tools.pillars import ADVICE_KEY + + await delivered(tmp_path) + # A Baseline whose design advisor consulted two pillars. + folder = tmp_path / "deliverables" / "cases" / CASE / "executions" / "v1" + advice = {"security": "Use a dedicated identity.", "reliability": "Two zones."} + (folder / "working.json").write_text( + json.dumps({"design_recommendations": {ADVICE_KEY: advice}}) + ) + seen: dict = {} + + def advisor(task): + seen["pillars_to_consult"] = task.context["pillars_to_consult"] + return [sample_submission(K.DESIGN_RECOMMENDATIONS)] + + decider = FakeDecider( + {"security": Answer(0.3, 0.8), "reliability": Answer(1.9, 0.9)} + ) + runtime = FakeAgentRuntime(changed(**{"design-advisor": [advisor]})) + engine = ExecutionEngine( + runtime=runtime, + roles=ROLES, + recorder=InMemoryRecorder(), + checkpointer=InMemorySaver(), + output_root=tmp_path, + orchestrator=Router(NoDecider(), None, Thresholds()), + decider=decider, + ) + + result = await engine.start( + "Availability goes up.", + customer_case=CASE, + pre_approved=True, + execution_id="v2", + baseline="v1", + ) + + assert seen["pillars_to_consult"] == ["reliability"] + state, _, _ = decider.asked[0] + assert "Requirement NFR-1 changed: 99.99% availability." in state + # The advice is carried into the new Execution, and written for the next change. + assert result.state["working"]["design_recommendations"][ADVICE_KEY] == advice + assert json.loads((folder.parent / "v2" / "working.json").read_text()) == { + "design_recommendations": {ADVICE_KEY: advice} + } + + +async def test_a_first_consultation_is_given_no_hint(tmp_path): + seen: dict = {} + + def advisor(task): + seen["pillars_to_consult"] = task.context["pillars_to_consult"] + return [sample_submission(K.DESIGN_RECOMMENDATIONS)] + + from app.adk.fake_runtime import FakeDecider + + decider = FakeDecider({}) + turns = once() + turns["design-advisor"] = [advisor] + engine = ExecutionEngine( + runtime=FakeAgentRuntime(turns), + roles=ROLES, + recorder=InMemoryRecorder(), + checkpointer=InMemorySaver(), + output_root=tmp_path, + orchestrator=Router(NoDecider(), None, Thresholds()), + decider=decider, + ) + + await engine.start("Ingest webhooks", pre_approved=True) + + assert seen["pillars_to_consult"] is None and decider.asked == [] diff --git a/tests/execution/test_pillars.py b/tests/execution/test_pillars.py index bf831c1..7ff603c 100644 --- a/tests/execution/test_pillars.py +++ b/tests/execution/test_pillars.py @@ -408,3 +408,59 @@ async def test_with_no_review_nothing_is_added(): result = await submit.handler(**sample_submission(kind).args) assert REVIEW_HEADING not in result.outcome.content + + +# --- only the pillars a change touches are consulted again ------------------------------- + + +async def test_a_change_names_the_pillars_it_affects_directly(): + from app.adk.fake_runtime import FakeDecider + from app.adk.runtime import Answer + from app.tools.pillars import pillars_touched + + decider = FakeDecider( + { + "security": Answer(1.9, 0.9), + "reliability": Answer(1.0, 0.6), + "sustainability": Answer(0.2, 0.8), + } + ) + + chosen = await pillars_touched( + decider, list(PILLARS), "Use customer-managed keys.", scope="e1" + ) + + state, questions, scope = decider.asked[0] + assert chosen == ["security"] + assert scope == "e1" and state.endswith("Use customer-managed keys.") + assert questions["security"].kind == "score" + assert questions["security"].options == ( + "not at all", + "only indirectly", + "directly", + ) + + +async def test_a_change_that_touches_no_pillar_is_an_answer_and_no_decision_is_not(): + from app.adk.fake_runtime import FakeDecider + from app.adk.runtime import Answer, NoDecider + from app.tools.pillars import pillars_touched + + nothing = FakeDecider(dict.fromkeys(PILLARS, Answer(0.0, 0.99))) + + assert await pillars_touched(nothing, list(PILLARS), "Rename a page.") == [] + assert await pillars_touched(NoDecider(), list(PILLARS), "Rename a page.") is None + assert await pillars_touched(nothing, list(PILLARS), " ") is None + + +async def test_an_empty_list_of_pillars_consults_none_and_returns_what_was_said(): + memory: dict = {} + runtime = FakeAgentRuntime({PILLAR_ADVISOR: [turn_for_pillar] * 3}) + tool = pillar_consultation_tool(runtime, ADVISOR, PILLARS, memory=memory) + await consult(tool, brief="b") + + again = await consult(tool, brief="b", pillars=[]) + + assert len(runtime.runs) == 3 # nobody was consulted a second time + for pillar in PILLARS: + assert f"## {pillar} (from your earlier consultation)" in again.message