feat: a stated choice must reach a requirement, and only pinned providers (iteration 9 step 3)
Some checks failed
validation / verify (push) Failing after 11s
Some checks failed
validation / verify (push) Failing after 11s
Case 08 named the customer's stated topology in its design and did not build it; every Check passed because none asked for it. - The catalogue marks the 26 fields that bind a design. The agreed scope lists the ones the customer stated; what was assumed for them does not bind. - A requirement says what it comes from. A submission of requirements is refused when one of the customer's own requirements, or a stated binding field, is named by none, or when the requirement carrying a choice is not a constraint with a condition. The existing chain then owes it a Check. - Generated code may use only the providers the bundle pins: a resource or a required_providers entry of any other is refused before Terraform runs. - The guide opens with how much of the design the customer stated, how much was assumed and how much is unknown. Not run with real models. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -767,3 +767,113 @@ def test_a_constraint_with_conditions_still_needs_a_check():
|
||||
)
|
||||
|
||||
assert gaps == ("Requirement NFR-2 has no Check",)
|
||||
|
||||
|
||||
# --- what the customer stated reaches the requirements (ADR-0008) ------------------------
|
||||
|
||||
|
||||
def scoped_upstream(**scope) -> dict:
|
||||
import json
|
||||
|
||||
return {"scope": {"attachments": {"scope.json": json.dumps(scope)}}}
|
||||
|
||||
|
||||
def requirement_problems(requirements: list[dict], **scope) -> list[str]:
|
||||
import json
|
||||
|
||||
from app.workflows.chain import check_requirements
|
||||
|
||||
return check_requirements(
|
||||
{"requirements.json": json.dumps(requirements)}, scoped_upstream(**scope)
|
||||
)
|
||||
|
||||
|
||||
STATED = {
|
||||
"stated": {
|
||||
"intent": {
|
||||
"requirements": [{"id": "REQ-01", "statement": "Ready in 10 weeks."}]
|
||||
},
|
||||
"networking": {"topology": "hub_and_spoke"},
|
||||
},
|
||||
"binding": ["networking.topology"],
|
||||
}
|
||||
FUNCTIONAL = {"id": "FR-1", "kind": "functional", "text": "Accept events."}
|
||||
CARRIED = {
|
||||
"id": "NFR-1",
|
||||
"kind": "non_functional",
|
||||
"text": "The network is hub and spoke, ready in ten weeks.",
|
||||
"conditions": ["A hub network is connected to each spoke network."],
|
||||
"from": ["REQ-01", "networking.topology"],
|
||||
}
|
||||
|
||||
|
||||
def test_requirements_that_carry_everything_the_customer_stated_are_accepted():
|
||||
assert requirement_problems([FUNCTIONAL, CARRIED], **STATED) == []
|
||||
|
||||
|
||||
def test_a_customers_own_requirement_must_be_covered_by_name():
|
||||
uncited = {**CARRIED, "from": ["networking.topology"]}
|
||||
|
||||
(problem,) = requirement_problems([FUNCTIONAL, uncited], **STATED)
|
||||
|
||||
assert "the customer's requirement REQ-01 is covered by no requirement" in problem
|
||||
|
||||
|
||||
def test_a_stated_choice_must_be_carried_by_a_requirement():
|
||||
"""Case 08: a stated topology was named in the design and not built, and every
|
||||
Check passed, because nothing asked for it."""
|
||||
uncarried = {**CARRIED, "from": ["REQ-01"]}
|
||||
|
||||
(problem,) = requirement_problems([FUNCTIONAL, uncarried], **STATED)
|
||||
|
||||
assert "the customer stated networking.topology and no requirement carries it" in (
|
||||
problem
|
||||
)
|
||||
|
||||
|
||||
def test_a_stated_choice_is_carried_as_a_constraint_the_design_is_checked_against():
|
||||
as_functional = {**CARRIED, "kind": "functional", "conditions": []}
|
||||
outside_code_only = {
|
||||
**CARRIED,
|
||||
"conditions": [],
|
||||
"outside_code": ["The topology is reviewed by the network team."],
|
||||
}
|
||||
|
||||
for weak in (as_functional, outside_code_only):
|
||||
problems = requirement_problems([FUNCTIONAL, weak], **STATED)
|
||||
assert any(
|
||||
"must be non_functional with at least one entry in `conditions`" in p
|
||||
for p in problems
|
||||
), weak
|
||||
|
||||
|
||||
def test_with_no_scope_or_nothing_binding_requirements_are_as_before():
|
||||
from app.workflows.chain import check_requirements
|
||||
|
||||
plain = [FUNCTIONAL]
|
||||
import json
|
||||
|
||||
assert check_requirements({"requirements.json": json.dumps(plain)}, None) == []
|
||||
assert requirement_problems(plain, stated={"intent": {"statement": "x"}}) == []
|
||||
|
||||
|
||||
def test_what_was_assumed_for_the_customer_does_not_bind_the_design():
|
||||
from app.scope import Catalogue, FieldSpec, Scope
|
||||
|
||||
catalogue = Catalogue(
|
||||
1,
|
||||
("workload",),
|
||||
{
|
||||
"net.topology": FieldSpec("net.topology", binds=True),
|
||||
"net.region": FieldSpec("net.region", binds=True),
|
||||
"notes": FieldSpec("notes"),
|
||||
},
|
||||
)
|
||||
scope = Scope(
|
||||
catalogue,
|
||||
"workload",
|
||||
values={"net.topology": "hub", "net.region": "x", "notes": "n"},
|
||||
sources={"net.region": "assumed"},
|
||||
)
|
||||
|
||||
assert scope.for_context()["binding"] == ["net.topology"]
|
||||
|
||||
@@ -14,6 +14,7 @@ from app.providers import load_provider
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH
|
||||
from app.workflows.deliverables import DeliverableKind as K
|
||||
from tests.scoped import carrying
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
@@ -45,6 +46,15 @@ run: {pre_approved: true, terraform_modules: false}
|
||||
"""
|
||||
|
||||
|
||||
STATED_CHOICES = [
|
||||
"estate.regions.primary",
|
||||
"estate.data_residency",
|
||||
"compliance.frameworks",
|
||||
"compliance.data_classification",
|
||||
"networking.topology",
|
||||
]
|
||||
|
||||
|
||||
class Recording(FakeAgentRuntime):
|
||||
def __init__(self, turns):
|
||||
super().__init__(turns)
|
||||
@@ -62,8 +72,13 @@ def engine(tmp_path) -> tuple[ExecutionEngine, Recording]:
|
||||
bundle = load_provider("gcp")
|
||||
runtime = Recording(
|
||||
{
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
**{
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
},
|
||||
# The requirements carry every choice SPEC states, and a Check holds
|
||||
# the design to them.
|
||||
**carrying(STATED_CHOICES),
|
||||
}
|
||||
)
|
||||
built = ExecutionEngine(
|
||||
|
||||
@@ -78,3 +78,61 @@ def scoped_ledger(**options: Any): # type: ignore[no-untyped-def]
|
||||
if ledger.phase_status(PhaseId.SCOPING) is PhaseStatus.AWAITING_APPROVAL:
|
||||
ledger.approve(PhaseId.SCOPING, actor="user")
|
||||
return ledger
|
||||
|
||||
|
||||
def carrying(sources: list[str]) -> dict[str, list]:
|
||||
"""Scripted turns for the Specialists along the Requirements chain, in which one
|
||||
more requirement, NFR-2, carries what the customer stated (`sources`), and a
|
||||
Check, K-3, holds the design to it."""
|
||||
import json
|
||||
|
||||
from app.adk.fake_runtime import SAMPLE_DATA, Call
|
||||
|
||||
def submission(kind: DeliverableKind, **files: str) -> Call:
|
||||
call = sample_submission(kind)
|
||||
return Call(
|
||||
call.tool,
|
||||
{**call.args, "attachments": {**call.args["attachments"], **files}},
|
||||
)
|
||||
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
requirements.append(
|
||||
{
|
||||
"id": "NFR-2",
|
||||
"kind": "non_functional",
|
||||
"text": "The design honours what the customer stated.",
|
||||
"conditions": ["Each stated choice is configured as stated."],
|
||||
"from": sources,
|
||||
}
|
||||
)
|
||||
components = json.loads(SAMPLE_DATA["components.json"])
|
||||
components[0]["requirements"].append("NFR-2")
|
||||
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
graph["checks"].append(
|
||||
{
|
||||
"id": "K-3",
|
||||
"applies_to": "E-1",
|
||||
"source": "NFR-2",
|
||||
"statement": "the stated choices are configured",
|
||||
}
|
||||
)
|
||||
implementation = json.loads(SAMPLE_DATA["implementation.json"])
|
||||
implementation.append(
|
||||
{"check": "K-3", "file": "terraform/main.tf", "address": "example.run"}
|
||||
)
|
||||
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
verdicts["checks"].append(
|
||||
{"check": "K-3", "verdict": "pass", "evidence": "Configured as stated."}
|
||||
)
|
||||
K = DeliverableKind
|
||||
files = {
|
||||
K.REQUIREMENTS: {"requirements.json": json.dumps(requirements)},
|
||||
K.TECHNICAL_DECOMPOSITION: {"components.json": json.dumps(components)},
|
||||
K.TARGET_ARCHITECTURE: {"target-architecture.json": json.dumps(graph)},
|
||||
K.DEPLOYMENT_GUIDANCE: {"implementation.json": json.dumps(implementation)},
|
||||
K.VALIDATION_RESULTS: {"verdicts.json": json.dumps(verdicts)},
|
||||
}
|
||||
return {
|
||||
DEPENDENCY_GRAPH[kind].specialist: [[submission(kind, **attached)]]
|
||||
for kind, attached in files.items()
|
||||
}
|
||||
|
||||
@@ -383,3 +383,72 @@ def test_the_cache_setting_resolves_under_the_repository_and_can_be_disabled():
|
||||
).terraform_plugin_cache()
|
||||
== "/var/cache/tf"
|
||||
)
|
||||
|
||||
|
||||
# --- only the providers the bundle pins ---------------------------------------------------
|
||||
|
||||
PINNED = {"cloud": {"source": "example/cloud", "version": "1.2.3"}}
|
||||
|
||||
|
||||
def provider_refusal(code: str, pinned=PINNED) -> str:
|
||||
from app.tools.local import check_providers
|
||||
|
||||
result = check_providers({"terraform/main.tf": code}, pinned)
|
||||
return result.message if result else ""
|
||||
|
||||
|
||||
def test_code_of_the_pinned_provider_alone_is_accepted():
|
||||
code = """
|
||||
terraform {
|
||||
required_providers {
|
||||
cloud = {
|
||||
source = "example/cloud"
|
||||
version = "1.2.3"
|
||||
}
|
||||
}
|
||||
}
|
||||
resource "cloud_service" "web" {}
|
||||
data "cloud_project" "this" {}
|
||||
resource "terraform_data" "marker" {}
|
||||
module "network" {
|
||||
source = "example-modules/network/cloud"
|
||||
version = "1.0.0"
|
||||
}
|
||||
"""
|
||||
|
||||
assert provider_refusal(code) == ""
|
||||
|
||||
|
||||
def test_a_resource_of_a_provider_that_is_not_pinned_is_refused():
|
||||
message = provider_refusal('resource "sourcehost_repository" "infra" {}')
|
||||
|
||||
assert message.startswith("Refused: the code uses resources of the sourcehost")
|
||||
assert "Only these providers may be used: example/cloud" in message
|
||||
|
||||
|
||||
def test_declaring_another_provider_is_refused():
|
||||
declared = """
|
||||
terraform {
|
||||
required_providers {
|
||||
sourcehost = {
|
||||
source = "integrations/sourcehost"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
assert "the provider integrations/sourcehost" in provider_refusal(declared)
|
||||
|
||||
|
||||
def test_with_nothing_pinned_there_is_nothing_to_hold_code_to():
|
||||
assert provider_refusal('resource "anything_at_all" "x" {}', pinned={}) == ""
|
||||
|
||||
|
||||
async def test_validation_refuses_a_foreign_provider_before_terraform_runs():
|
||||
runner = TerraformRunner(binary="no-such-binary", providers=PINNED)
|
||||
|
||||
result = await runner.validate(
|
||||
{"terraform/main.tf": 'resource "sourcehost_repository" "infra" {}'}
|
||||
)
|
||||
|
||||
assert result.message.startswith("Refused: the code uses")
|
||||
|
||||
Reference in New Issue
Block a user