Files
gcp_solution_architecture_a…/tests/execution/test_validation_in_pieces.py
Jonathan Boniface 58e778ca7a feat: record verdicts in pieces and check the design against the schema
From the case 02 run cancelled at 66 minutes on 2026-10-07:

- The validator records verdicts a few Checks at a time (verdict_put) and
  they are attached on submission. It timed out five times writing them in
  one reply. Verdicts on the same code survive a run cut short; verdicts on
  changed code are dropped. Its model timeout is 300 s.
- A schema review at submission, in code: every setting a Check names is
  looked up on its Element's resource type, and an unknown resource type is
  reported. Two Revisions in that run were for Checks no code could meet.
  On the two designs that passed validation it raises 2 of 96 Checks, both
  one metric name used as a value, so it is returned once and never blocks.
- Reviews run together; the Jev coverage review stays off.

Plans record both steps and the cancelled run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 09:39:13 +01:00

280 lines
9.6 KiB
Python

"""Two fixes from the run cancelled on 2026-10-07.
The validator wrote every verdict in one reply and timed out five times; it now
records them a few at a time. And two Checks named settings their resource types do
not have, each costing a Revision; the design is now checked against the provider's
schema, in code, before it is submitted.
"""
import json
from app.adk.fake_runtime import SAMPLE_DATA, sample_submission
from app.adk.runtime import Submitted, ToolResult
from app.tools.execution import specialist_tools
from app.tools.local import TerraformRunner
from app.tools.schema_review import Reviews, SchemaReview, named_settings
from app.workflows.deliverables import DeliverableKind
K = DeliverableKind
CODE = {"terraform/main.tf": "# v1\n", "implementation.json": "[]"}
VERDICTS = json.loads(SAMPLE_DATA["verdicts.json"])
CONTENT = sample_submission(K.VALIDATION_RESULTS).args["content"]
def upstream(code=CODE) -> dict:
return {
"target_architecture": {
"attachments": {
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
}
},
"deployment_guidance": {"attachments": dict(code)},
}
def validator_tools(working: dict, code=CODE) -> dict:
return {
tool.name: tool
for tool in specialist_tools(
K.VALIDATION_RESULTS,
(),
allow_cloud_read=False,
upstream=upstream(code),
working=working,
)
}
# --- verdicts a few at a time ------------------------------------------------------------
async def test_verdicts_are_recorded_a_few_at_a_time_and_attached_on_submission():
working: dict = {}
tools = validator_tools(working)
first = await tools["verdict_put"].handler(
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
)
second = await tools["verdict_put"].handler(checks=VERDICTS["checks"][1:])
result = await tools["submit_deliverable"].handler(content=CONTENT)
assert first.message == "Recorded. 1 of 2 Checks have a verdict. Left: K-2."
assert second.message == "Recorded. All 2 Checks have a verdict."
assert isinstance(result.outcome, Submitted)
attached = json.loads(result.outcome.attachments["verdicts.json"])
assert [item["check"] for item in attached["checks"]] == ["K-1", "K-2"]
assert attached["dry_run"]["verdict"] == "pass"
async def test_a_submission_with_a_check_left_out_names_it():
tools = validator_tools({})
await tools["verdict_put"].handler(
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
)
result = await tools["submit_deliverable"].handler(content=CONTENT)
assert result.outcome is None
assert "Check K-2 is not accounted for" in result.message
async def test_a_verdict_given_again_replaces_the_first_and_additions_are_kept():
working: dict = {}
tools = validator_tools(working)
addition = {
"statement": "A timeout is set.",
"applies_to": "E-1",
"reason": "NFR-1",
}
await tools["verdict_put"].handler(checks=VERDICTS["checks"])
await tools["verdict_put"].handler(
checks=[{"check": "K-1", "verdict": "fail", "evidence": "min is 1"}],
proposed_additions=[addition, addition],
)
read = await tools["verdict_read"].handler()
recorded = json.loads(read.message.rsplit("\n", 1)[0])
assert [(c["check"], c["verdict"]) for c in recorded["checks"]] == [
("K-1", "fail"),
("K-2", "pass"),
]
assert recorded["proposed_additions"] == [addition]
assert read.message.endswith("All 2 Checks have a verdict.")
async def test_a_run_cut_short_carries_on_from_what_it_recorded():
working: dict = {}
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"][:1])
# The same code, a later attempt: the working memory came back with the Execution.
read = await validator_tools(working)["verdict_read"].handler()
assert "1 of 2 Checks have a verdict. Left: K-2." in read.message
async def test_verdicts_on_code_that_has_changed_are_dropped():
working: dict = {}
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"])
revised = {**CODE, "terraform/main.tf": "# v2\n"}
read = await validator_tools(working, revised)["verdict_read"].handler()
assert read.message.startswith("No verdicts recorded yet.")
async def test_verdicts_passed_as_an_attachment_still_work():
call = sample_submission(K.VALIDATION_RESULTS)
result = await validator_tools({})["submit_deliverable"].handler(**call.args)
assert isinstance(result.outcome, Submitted)
def test_only_the_validator_records_verdicts():
assert "verdict_put" in validator_tools({})
assert "verdict_put" not in {
tool.name
for tool in specialist_tools(K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False)
}
# --- the design against the provider's schema --------------------------------------------
def test_the_settings_a_check_names_are_read_from_its_statement():
assert named_settings("`kms_key_name` is set on the topic") == ("kms_key_name",)
assert named_settings("template.scaling.min_instance_count is at least 2") == (
"template.scaling.min_instance_count",
)
assert named_settings("startup_cpu_boost is true") == ("startup_cpu_boost",)
# Values, roles and plain prose name no setting.
assert named_settings("ingress is INGRESS_TRAFFIC_ALL") == ()
assert named_settings("roles/run.invoker is granted to allUsers") == ()
assert named_settings('the filter contains "oldest_unacked_message_age"') == ()
assert named_settings("The service is private.") == ()
SCHEMA = {
"cloud_service": (
"example/cloud 1.2.3",
{
"attributes": {"name": {}, "ingress": {}},
"block_types": {
"template": {
"block": {
"attributes": {"service_account": {}},
"block_types": {
"scaling": {
"block": {"attributes": {"min_instance_count": {}}}
}
},
}
}
},
},
),
"cloud_job": ("example/cloud 1.2.3", {"attributes": {"name": {}}}),
"cloud_service_iam_member": ("example/cloud 1.2.3", {"attributes": {"role": {}}}),
}
class Runner(TerraformRunner):
def __init__(self, schemas=SCHEMA) -> None:
super().__init__()
self._given = schemas
async def _load_schemas(self):
return self._given
def graph(checks, resource_types=("cloud_service", "cloud_job")) -> dict[str, str]:
elements = [
{
"id": f"E-{n}",
"name": f"Thing {n}",
"product": "Example",
"resource_type": kind,
}
for n, kind in enumerate(resource_types, start=1)
]
data = {
"elements": elements,
"relationships": [{"id": "R-1", "from": "E-1", "to": "E-2", "kind": "calls"}],
"checks": [
{"id": f"K-{n}", "applies_to": on, "statement": text, "source": "NFR-1"}
for n, (on, text) in enumerate(checks, start=1)
],
}
return {"target-architecture.json": json.dumps(data)}
async def test_a_check_naming_a_setting_its_resource_type_lacks_is_raised():
design = graph(
[
("E-1", "template.scaling.min_instance_count is at least 2"),
("E-2", "`min_instance_count` is at least 2"),
("E-2", "the name is set"),
]
)
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
assert len(raised) == 1
assert raised[0].startswith(
"Check K-2: cloud_job has no setting named min_instance_count"
)
async def test_a_setting_found_anywhere_in_the_resource_type_is_accepted():
design = graph(
[
("E-1", "service_account is the dedicated identity"),
("E-1", "ingress is INTERNAL"),
("E-1", "a cloud_service_iam_member grants the invoker role"),
("R-1", "any_made_up_setting is checked on a Relationship"),
("E-2", "the name is set"),
]
)
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) == ()
async def test_a_resource_type_the_provider_lacks_is_raised_with_close_matches():
design = graph(
[("E-1", "the name is set"), ("E-2", "the name is set")],
resource_types=("cloud_servce", "cloud_job"),
)
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
assert len(raised) == 1
assert raised[0].startswith(
"Element E-1: cloud_servce is not a resource type of the provider."
)
assert "Close matches: cloud_service" in raised[0]
async def test_no_schema_and_other_deliverables_raise_nothing():
design = graph([("E-1", "made_up_thing is set"), ("E-2", "the name is set")])
unavailable = Runner(ToolResult("Schema lookup is unavailable."))
assert await SchemaReview(unavailable)(K.TARGET_ARCHITECTURE, design, None) == ()
assert await SchemaReview(Runner())(K.DEPLOYMENT_GUIDANCE, design, None) == ()
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, {}, None) == ()
async def test_several_reviews_raise_together_and_one_failing_loses_nothing():
async def one(kind, attachments, upstream, scope=""):
return ["first"]
async def broken(kind, attachments, upstream, scope=""):
raise RuntimeError("down")
async def two(kind, attachments, upstream, scope=""):
return ("second",)
raised = await Reviews([one, broken, two])(K.TARGET_ARCHITECTURE, {}, None)
assert raised == ("first", "second")