After deployment guidance was reopened, the Orchestrator's context still carried the previous validation's verdicts, so it reopened the code for the same Check three times without a validation in between (evaluation run of 2026-10-07, case 01). Record that run in the iteration 6 plan. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
620 lines
21 KiB
Python
620 lines
21 KiB
Python
"""The Requirements chain: the Target architecture graph and the data files around it.
|
|
|
|
ADR-0005: the outcome is understood before code is written, and validation answers
|
|
the graph's Checks and nothing else.
|
|
"""
|
|
|
|
import json
|
|
|
|
import pytest
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from app.adk.fake_runtime import SAMPLE_DATA, Call, FakeAgentRuntime, sample_submission
|
|
from app.adk.runtime import RoleSpec
|
|
from app.agent import ExecutionEngine
|
|
from app.recorder import InMemoryRecorder
|
|
from app.tools.execution import specialist_tools
|
|
from app.tools.local import mermaid_check
|
|
from app.workflows.chain import validation_outcome
|
|
from app.workflows.deliverables import (
|
|
DEPENDENCY_GRAPH,
|
|
DeliverableKind,
|
|
PhaseId,
|
|
check_submission,
|
|
)
|
|
from app.workflows.target_architecture import (
|
|
GraphError,
|
|
chain_gaps,
|
|
parse_graph,
|
|
render_mermaid,
|
|
)
|
|
|
|
K = DeliverableKind
|
|
P = PhaseId
|
|
|
|
REQUIREMENTS = json.loads(SAMPLE_DATA["requirements.json"])
|
|
COMPONENTS = json.loads(SAMPLE_DATA["components.json"])
|
|
PRODUCTS = json.loads(SAMPLE_DATA["products.json"])
|
|
RECOMMENDATIONS = json.loads(SAMPLE_DATA["recommendations.json"])
|
|
|
|
|
|
def graph_data() -> dict:
|
|
return json.loads(SAMPLE_DATA["target-architecture.json"])
|
|
|
|
|
|
def problems(data: dict) -> tuple[str, ...]:
|
|
with pytest.raises(GraphError) as raised:
|
|
parse_graph(json.dumps(data))
|
|
return raised.value.problems
|
|
|
|
|
|
def gaps(data: dict, **overrides) -> tuple[str, ...]:
|
|
links = {
|
|
"requirements": REQUIREMENTS,
|
|
"components": COMPONENTS,
|
|
"products": PRODUCTS,
|
|
"recommendations": RECOMMENDATIONS,
|
|
**overrides,
|
|
}
|
|
return chain_gaps(parse_graph(json.dumps(data)), **links)
|
|
|
|
|
|
def upstream(*kinds: DeliverableKind, **replaced: str) -> dict:
|
|
"""The Deliverables a submission builds on, from the coherent sample."""
|
|
built = {}
|
|
for kind in kinds:
|
|
args = sample_submission(kind).args
|
|
attachments = {
|
|
name: replaced.get(name, text) for name, text in args["attachments"].items()
|
|
}
|
|
built[kind.value] = {"content": args["content"], "attachments": attachments}
|
|
return built
|
|
|
|
|
|
DESIGN = (
|
|
K.REQUIREMENTS,
|
|
K.TECHNICAL_DECOMPOSITION,
|
|
K.PRODUCT_SELECTION,
|
|
K.DESIGN_RECOMMENDATIONS,
|
|
)
|
|
|
|
|
|
# --- the graph is well-formed ----------------------------------------------------------
|
|
|
|
|
|
def test_the_sample_graph_parses_into_elements_relationships_and_checks():
|
|
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
|
|
|
|
assert [element.id for element in graph.elements] == ["E-1", "E-2"]
|
|
assert graph.elements[1].component is None # a supporting Element
|
|
assert [(r.source, r.target) for r in graph.relationships] == [("E-1", "E-2")]
|
|
assert graph.check_ids == ("K-1", "K-2")
|
|
assert graph.summary() == {
|
|
"elements": 2,
|
|
"relationships": 1,
|
|
"checks": 2,
|
|
"not_applicable": [],
|
|
}
|
|
|
|
|
|
def test_a_relationship_must_join_two_elements():
|
|
data = graph_data()
|
|
data["relationships"][0]["to"] = "E-9"
|
|
|
|
assert problems(data) == (
|
|
"target-architecture.json: Relationship R-1 to E-9, which is not an Element",
|
|
)
|
|
|
|
|
|
def test_a_check_must_apply_to_something_in_the_graph():
|
|
data = graph_data()
|
|
data["checks"][0]["applies_to"] = "E-9"
|
|
|
|
assert problems(data) == (
|
|
(
|
|
"target-architecture.json: Check K-1 applies to E-9, "
|
|
"which is not an Element or Relationship"
|
|
),
|
|
"target-architecture.json: Element E-1 has no Check",
|
|
)
|
|
|
|
|
|
def test_every_element_needs_a_check_and_every_id_is_used_once():
|
|
data = graph_data()
|
|
data["checks"] = [data["checks"][0]]
|
|
data["relationships"][0]["id"] = "E-1"
|
|
|
|
assert problems(data) == (
|
|
"target-architecture.json: id E-1 is used more than once",
|
|
"target-architecture.json: Element E-2 has no Check",
|
|
)
|
|
|
|
|
|
def test_a_check_without_a_statement_is_named():
|
|
data = graph_data()
|
|
data["checks"][1]["statement"] = " "
|
|
|
|
assert (
|
|
"target-architecture.json.checks[1]: needs a non-empty statement"
|
|
in problems(data)
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("text", ["not json", "[]", '{"elements": "many"}'])
|
|
def test_a_graph_that_is_not_the_right_shape_is_refused(text):
|
|
with pytest.raises(GraphError):
|
|
parse_graph(text)
|
|
|
|
|
|
# --- the Requirements chain has no gap -------------------------------------------------
|
|
|
|
|
|
def test_the_sample_chain_is_complete():
|
|
assert gaps(graph_data()) == ()
|
|
|
|
|
|
def test_a_non_functional_requirement_with_no_check_is_a_gap():
|
|
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
|
|
|
|
assert gaps(graph_data(), requirements=[*REQUIREMENTS, retention]) == (
|
|
"Requirement NFR-2 has no Check",
|
|
)
|
|
|
|
|
|
def test_a_functional_requirement_is_carried_by_its_components_element():
|
|
# FR-1 is the source of no Check, but C-1 serves it and E-1 realises C-1.
|
|
assert not [gap for gap in gaps(graph_data()) if "FR-1" in gap]
|
|
|
|
unserved = [{**COMPONENTS[0], "requirements": ["NFR-1"]}]
|
|
assert gaps(graph_data(), components=unserved) == ("Requirement FR-1 has no Check",)
|
|
|
|
|
|
def test_a_component_with_no_element_is_a_gap():
|
|
replay = {"id": "C-2", "name": "Replay service", "requirements": []}
|
|
|
|
assert gaps(graph_data(), components=[*COMPONENTS, replay]) == (
|
|
"Component C-2 has no Element",
|
|
)
|
|
|
|
|
|
def test_a_selected_product_that_no_element_uses_is_a_gap():
|
|
armour = [{"component": "C-1", "product": "Example Armor"}]
|
|
|
|
assert gaps(graph_data(), products=armour) == (
|
|
"Product Example Armor selected for C-1 appears on no Element",
|
|
)
|
|
|
|
|
|
def test_a_recommendation_needs_a_check_or_a_reason_it_does_not_apply():
|
|
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
|
|
recommendations = [*RECOMMENDATIONS, cmek]
|
|
|
|
assert gaps(graph_data(), recommendations=recommendations) == (
|
|
"Recommendation DR-2 has no Check",
|
|
)
|
|
|
|
excused = graph_data()
|
|
excused["not_applicable"] = [
|
|
{"recommendation": "DR-2", "reason": "No regulated data."}
|
|
]
|
|
assert gaps(excused, recommendations=recommendations) == ()
|
|
assert parse_graph(json.dumps(excused)).summary()["not_applicable"] == [
|
|
{"recommendation": "DR-2", "reason": "No regulated data."}
|
|
]
|
|
|
|
|
|
def test_a_check_must_come_from_a_requirement_or_recommendation():
|
|
data = graph_data()
|
|
data["checks"][0]["source"] = "my own idea"
|
|
|
|
assert gaps(data) == (
|
|
(
|
|
"Check K-1 has source my own idea, which is not a requirement or design "
|
|
"recommendation"
|
|
),
|
|
"Requirement NFR-1 has no Check",
|
|
)
|
|
|
|
|
|
def test_a_target_architecture_with_a_gap_cannot_be_submitted():
|
|
retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."}
|
|
built_on = upstream(
|
|
*DESIGN, **{"requirements.json": json.dumps([*REQUIREMENTS, retention])}
|
|
)
|
|
args = sample_submission(K.TARGET_ARCHITECTURE).args
|
|
|
|
assert check_submission(
|
|
K.TARGET_ARCHITECTURE, args["content"], args["attachments"], built_on
|
|
) == ("Requirement NFR-2 has no Check",)
|
|
assert (
|
|
check_submission(
|
|
K.TARGET_ARCHITECTURE,
|
|
args["content"],
|
|
args["attachments"],
|
|
upstream(*DESIGN),
|
|
)
|
|
== ()
|
|
)
|
|
|
|
|
|
# --- the links before the graph --------------------------------------------------------
|
|
|
|
|
|
def test_requirements_need_ids_and_a_kind():
|
|
def missing(items) -> tuple[str, ...]:
|
|
args = sample_submission(K.REQUIREMENTS).args
|
|
return check_submission(
|
|
K.REQUIREMENTS, args["content"], {"requirements.json": json.dumps(items)}
|
|
)
|
|
|
|
assert missing(REQUIREMENTS) == ()
|
|
assert missing([]) == ("requirements.json: lists no requirements",)
|
|
assert missing([REQUIREMENTS[0], {**REQUIREMENTS[0], "kind": "must"}]) == (
|
|
"requirements.json: id FR-1 is used more than once",
|
|
"requirements.json: FR-1 kind must be one of functional, non_functional",
|
|
)
|
|
assert missing([{"id": "FR-1", "kind": "functional"}]) == (
|
|
"requirements.json[0]: needs text",
|
|
)
|
|
|
|
|
|
def test_every_functional_requirement_is_served_by_a_component():
|
|
args = sample_submission(K.TECHNICAL_DECOMPOSITION).args
|
|
|
|
def missing(components) -> tuple[str, ...]:
|
|
return check_submission(
|
|
K.TECHNICAL_DECOMPOSITION,
|
|
args["content"],
|
|
{"components.json": json.dumps(components)},
|
|
upstream(K.REQUIREMENTS),
|
|
)
|
|
|
|
assert missing(COMPONENTS) == ()
|
|
assert missing([{"id": "C-1", "name": "Ingress", "requirements": ["NFR-9"]}]) == (
|
|
"components.json: C-1 serves NFR-9, which is not a requirement",
|
|
"components.json: functional requirement FR-1 is served by no component",
|
|
)
|
|
|
|
|
|
def test_every_component_gets_a_product_and_recommendations_name_real_components():
|
|
selection = sample_submission(K.PRODUCT_SELECTION).args
|
|
advice = sample_submission(K.DESIGN_RECOMMENDATIONS).args
|
|
built_on = upstream(K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION)
|
|
|
|
assert check_submission(
|
|
K.PRODUCT_SELECTION,
|
|
selection["content"],
|
|
{"products.json": json.dumps([{"component": "C-7", "product": "X"}])},
|
|
built_on,
|
|
) == (
|
|
"products.json: C-7 is not a component",
|
|
"products.json: component C-1 has no product selected",
|
|
)
|
|
wandering = [{**RECOMMENDATIONS[0], "components": ["C-7"]}]
|
|
assert check_submission(
|
|
K.DESIGN_RECOMMENDATIONS,
|
|
advice["content"],
|
|
{"recommendations.json": json.dumps(wandering)},
|
|
built_on,
|
|
) == ("recommendations.json: DR-1 names C-7, which is not a component",)
|
|
|
|
|
|
# --- code and validation answer the graph ----------------------------------------------
|
|
|
|
EVERYTHING_BEFORE_CODE = (*DESIGN, K.TARGET_ARCHITECTURE)
|
|
|
|
|
|
def test_deployment_guidance_must_account_for_every_check():
|
|
args = sample_submission(K.DEPLOYMENT_GUIDANCE).args
|
|
partial = json.dumps(
|
|
[
|
|
{"check": "K-1", "file": "terraform/main.tf", "address": "a"},
|
|
{"check": "K-7", "file": "terraform/elsewhere.tf", "address": "b"},
|
|
]
|
|
)
|
|
|
|
missing = check_submission(
|
|
K.DEPLOYMENT_GUIDANCE,
|
|
args["content"],
|
|
{**args["attachments"], "implementation.json": partial},
|
|
upstream(*EVERYTHING_BEFORE_CODE),
|
|
)
|
|
|
|
assert missing == (
|
|
"implementation.json: K-7 is not a Check in the Target architecture",
|
|
"implementation.json: Check K-2 is not accounted for",
|
|
"implementation.json: K-7 names terraform/elsewhere.tf, which is not attached",
|
|
)
|
|
|
|
|
|
def verdicts(**changes) -> str:
|
|
data = json.loads(SAMPLE_DATA["verdicts.json"])
|
|
data.update(changes)
|
|
return json.dumps(data)
|
|
|
|
|
|
def validation_missing(text: str) -> tuple[str, ...]:
|
|
args = sample_submission(K.VALIDATION_RESULTS).args
|
|
return check_submission(
|
|
K.VALIDATION_RESULTS,
|
|
args["content"],
|
|
{"verdicts.json": text},
|
|
upstream(*EVERYTHING_BEFORE_CODE, K.DEPLOYMENT_GUIDANCE),
|
|
)
|
|
|
|
|
|
def test_validation_gives_exactly_one_verdict_per_check():
|
|
one = {"check": "K-1", "verdict": "pass", "evidence": "seen"}
|
|
invented = {"check": "K-9", "verdict": "fail", "evidence": "No Cloud Armor rule."}
|
|
|
|
assert validation_missing(SAMPLE_DATA["verdicts.json"]) == ()
|
|
assert validation_missing(verdicts(checks=[one, one, invented])) == (
|
|
"verdicts.json.checks: check K-1 is used more than once",
|
|
"verdicts.json.checks: K-9 is not a Check in the Target architecture",
|
|
"verdicts.json.checks: Check K-2 is not accounted for",
|
|
)
|
|
|
|
|
|
def test_the_validator_cannot_fail_the_code_on_something_the_graph_lacks():
|
|
"""What it notices beyond the Checks is a proposed addition, never a failure."""
|
|
addition = {
|
|
"statement": "Cloud Armor rate limiting is configured.",
|
|
"applies_to": "E-1",
|
|
"reason": "NFR-1",
|
|
}
|
|
text = verdicts(proposed_additions=[addition])
|
|
|
|
assert validation_missing(text) == ()
|
|
outcome = validation_outcome(text)
|
|
assert outcome["failed"] == []
|
|
assert outcome["proposed_additions"] == [addition]
|
|
|
|
|
|
def test_a_verdict_needs_evidence_and_a_known_value_and_the_dry_run_a_result():
|
|
vague = [
|
|
{"check": "K-1", "verdict": "probably", "evidence": "x"},
|
|
{"check": "K-2", "verdict": "pass", "evidence": " "},
|
|
]
|
|
|
|
assert validation_missing(verdicts(dry_run={"verdict": "ok"}, checks=vague)) == (
|
|
"verdicts.json.dry_run: needs a verdict (pass, fail, not_run) and evidence",
|
|
"verdicts.json.checks[1]: needs evidence",
|
|
"verdicts.json.checks: K-1 verdict must be one of pass, fail, not_verifiable",
|
|
"verdicts.json.checks: Check K-2 is not accounted for",
|
|
)
|
|
|
|
|
|
def test_a_validation_outcome_separates_faults_in_code_from_gaps_in_the_graph():
|
|
checks = [
|
|
{"check": "K-1", "verdict": "fail", "evidence": "min instances is 1"},
|
|
{"check": "K-2", "verdict": "not_verifiable", "evidence": "No such argument."},
|
|
]
|
|
|
|
outcome = validation_outcome(verdicts(checks=checks))
|
|
|
|
assert outcome["failed"] == [{"check": "K-1", "evidence": "min instances is 1"}]
|
|
assert outcome["not_verifiable"] == [
|
|
{"check": "K-2", "evidence": "No such argument."}
|
|
]
|
|
assert outcome["passed"] == []
|
|
assert outcome["dry_run"]["verdict"] == "pass"
|
|
|
|
|
|
# --- the diagram is a view of the graph ------------------------------------------------
|
|
|
|
|
|
async def test_the_diagram_shows_every_element_and_relationship_once():
|
|
graph = parse_graph(SAMPLE_DATA["target-architecture.json"])
|
|
|
|
diagram = render_mermaid(graph)
|
|
|
|
assert diagram == render_mermaid(graph) # deterministic
|
|
assert diagram.count("Ingress service") == 1
|
|
assert diagram.count("Ingress identity") == 1
|
|
assert diagram.count("-->") == 1
|
|
assert "runs as" in diagram
|
|
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
|
|
|
|
|
|
async def test_names_with_characters_mermaid_reserves_still_render():
|
|
data = graph_data()
|
|
data["elements"][0]["name"] = 'Ingest "edge" [v2] (public) | {x}'
|
|
data["elements"][0]["id"] = "svc/ingest.v2"
|
|
data["relationships"][0]["from"] = "svc/ingest.v2"
|
|
data["checks"][0]["applies_to"] = "svc/ingest.v2"
|
|
data["relationships"][0]["kind"] = "calls (HTTPS) | signed"
|
|
|
|
diagram = render_mermaid(parse_graph(json.dumps(data)))
|
|
|
|
assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed")
|
|
assert "svc/ingest.v2" not in diagram
|
|
|
|
|
|
async def test_the_engine_draws_the_diagram_when_the_target_architecture_is_submitted():
|
|
submit = specialist_tools(
|
|
K.TARGET_ARCHITECTURE,
|
|
extra=[],
|
|
allow_cloud_read=False,
|
|
upstream=upstream(*DESIGN),
|
|
)[0]
|
|
args = sample_submission(K.TARGET_ARCHITECTURE).args
|
|
|
|
result = await submit.handler(**args)
|
|
|
|
assert result.outcome.attachments["architecture.mmd"] == render_mermaid(
|
|
parse_graph(SAMPLE_DATA["target-architecture.json"])
|
|
)
|
|
|
|
|
|
async def test_a_gap_is_refused_to_the_specialist_with_the_missing_link_named():
|
|
cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []}
|
|
built_on = upstream(
|
|
*DESIGN, **{"recommendations.json": json.dumps([*RECOMMENDATIONS, cmek])}
|
|
)
|
|
submit = specialist_tools(
|
|
K.TARGET_ARCHITECTURE, extra=[], allow_cloud_read=False, upstream=built_on
|
|
)[0]
|
|
|
|
result = await submit.handler(**sample_submission(K.TARGET_ARCHITECTURE).args)
|
|
|
|
assert result.outcome is None
|
|
assert result.message == (
|
|
"Not submitted. The Deliverable is missing: Recommendation DR-2 has no Check"
|
|
)
|
|
|
|
|
|
# --- through the engine ----------------------------------------------------------------
|
|
|
|
ROLES = {
|
|
"orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40),
|
|
**{
|
|
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
|
|
|
|
def delegate(*kinds: DeliverableKind) -> Call:
|
|
return Call("delegate", {"kinds": [kind.value for kind in kinds]})
|
|
|
|
|
|
def approve(phase: PhaseId) -> Call:
|
|
return Call("request_approval", {"phase": phase.value, "summary": "done"})
|
|
|
|
|
|
def specialist(kind: DeliverableKind) -> str:
|
|
return DEPENDENCY_GRAPH[kind].specialist
|
|
|
|
|
|
def to_design_approval() -> list[list[Call]]:
|
|
return [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)],
|
|
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
|
[delegate(K.TARGET_ARCHITECTURE)],
|
|
]
|
|
|
|
|
|
def engine_for(tmp_path, turns: dict) -> tuple[ExecutionEngine, FakeAgentRuntime]:
|
|
runtime = FakeAgentRuntime(turns)
|
|
engine = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
)
|
|
return engine, runtime
|
|
|
|
|
|
async def test_the_user_approves_the_graph_before_any_code_is_written(tmp_path):
|
|
once = {
|
|
specialist(kind): [[sample_submission(kind)]]
|
|
for kind in (*DESIGN, K.TARGET_ARCHITECTURE)
|
|
}
|
|
engine, runtime = engine_for(
|
|
tmp_path,
|
|
{
|
|
**once,
|
|
"orchestrator": [
|
|
[delegate(K.REQUIREMENTS)],
|
|
[delegate(K.TECHNICAL_DECOMPOSITION)],
|
|
[approve(P.REQUIREMENTS_DISCOVERY)],
|
|
[delegate(K.PRODUCT_SELECTION)],
|
|
[delegate(K.DESIGN_RECOMMENDATIONS)],
|
|
[delegate(K.TARGET_ARCHITECTURE)],
|
|
# Tries to start the code before asking: refused, then asks.
|
|
[delegate(K.DEPLOYMENT_GUIDANCE), approve(P.SOLUTION_ARCHITECTURE)],
|
|
],
|
|
},
|
|
)
|
|
|
|
first = await engine.start("Ingest webhooks")
|
|
paused = await engine.resume(
|
|
first.execution_id,
|
|
{"approval": {"phase": "requirements_discovery", "decision": "approve"}},
|
|
)
|
|
|
|
assert paused.prompt["approval"]["phase"] == "solution_architecture"
|
|
assert paused.prompt["approval"]["target_architecture"] == {
|
|
"elements": 2,
|
|
"relationships": 1,
|
|
"checks": 2,
|
|
"not_applicable": [],
|
|
}
|
|
refused = runtime.runs[-1].results[0]
|
|
assert refused[0] == "delegate" and refused[1].startswith("Refused")
|
|
assert specialist(K.DEPLOYMENT_GUIDANCE) not in [run.role for run in runtime.runs]
|
|
|
|
|
|
async def test_the_orchestrator_is_told_what_validation_decided(tmp_path):
|
|
failing = json.loads(SAMPLE_DATA["verdicts.json"])
|
|
failing["checks"][0] = {
|
|
"check": "K-1",
|
|
"verdict": "fail",
|
|
"evidence": "min instances is 1",
|
|
}
|
|
failed_validation = sample_submission(K.VALIDATION_RESULTS)
|
|
failed_validation = Call(
|
|
failed_validation.tool,
|
|
{
|
|
**failed_validation.args,
|
|
"attachments": {"verdicts.json": json.dumps(failing)},
|
|
},
|
|
)
|
|
once = {
|
|
specialist(kind): [[sample_submission(kind)]]
|
|
for kind in (*DESIGN, K.TARGET_ARCHITECTURE, K.ARCHITECTURE_DESCRIPTION)
|
|
}
|
|
engine, runtime = engine_for(
|
|
tmp_path,
|
|
{
|
|
**once,
|
|
"orchestrator": [
|
|
*to_design_approval(),
|
|
[
|
|
approve(P.SOLUTION_ARCHITECTURE),
|
|
delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE),
|
|
],
|
|
[approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)],
|
|
[
|
|
Call(
|
|
"reopen",
|
|
{
|
|
"kind": "deployment_guidance",
|
|
"reason": "K-1 failed: min instances is 1",
|
|
},
|
|
),
|
|
delegate(K.DEPLOYMENT_GUIDANCE),
|
|
],
|
|
[],
|
|
],
|
|
specialist(K.DEPLOYMENT_GUIDANCE): [
|
|
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
|
[sample_submission(K.DEPLOYMENT_GUIDANCE)],
|
|
],
|
|
specialist(K.VALIDATION_RESULTS): [[failed_validation]],
|
|
},
|
|
)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
turns = [run for run in runtime.runs if run.role == "orchestrator"]
|
|
before, after = turns[6].task.context, turns[7].task.context
|
|
assert before["validation"] is None
|
|
assert after["validation"]["failed"] == [
|
|
{"check": "K-1", "evidence": "min instances is 1"}
|
|
]
|
|
assert after["validation"]["proposed_additions"] == []
|
|
# Once the code is reopened those verdicts are about work that no longer exists,
|
|
# so the next turn must not see them and reopen the same thing again.
|
|
assert turns[8].task.context["validation"] is None
|
|
# Revising the code leaves the approved design standing.
|
|
phases = result.state["ledger"]["phases"]
|
|
assert phases["solution_architecture"] == "approved"
|
|
revised = [
|
|
run for run in runtime.runs if run.role == specialist(K.DEPLOYMENT_GUIDANCE)
|
|
][1]
|
|
assert revised.task.context["feedback"] == ["K-1 failed: min instances is 1"]
|