feat: record verdicts in pieces and check the design against the schema
From the case 02 run cancelled at 66 minutes on 2026-10-07: - The validator records verdicts a few Checks at a time (verdict_put) and they are attached on submission. It timed out five times writing them in one reply. Verdicts on the same code survive a run cut short; verdicts on changed code are dropped. Its model timeout is 300 s. - A schema review at submission, in code: every setting a Check names is looked up on its Element's resource type, and an unknown resource type is reported. Two Revisions in that run were for Checks no code could meet. On the two designs that passed validation it raises 2 of 96 Checks, both one metric name used as a value, so it is returned once and never blocks. - Reviews run together; the Jev coverage review stays off. Plans record both steps and the cancelled run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -65,6 +65,7 @@ def test_roles_with_long_replies_are_given_longer_for_a_model_request(bundle):
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
|
||||
assert roles["target-architect"].model_timeout_seconds == 420
|
||||
assert roles["validator"].model_timeout_seconds == 300
|
||||
assert roles["decomposer"].model_timeout_seconds is None
|
||||
|
||||
|
||||
|
||||
279
tests/execution/test_validation_in_pieces.py
Normal file
279
tests/execution/test_validation_in_pieces.py
Normal file
@@ -0,0 +1,279 @@
|
||||
"""Two fixes from the run cancelled on 2026-10-07.
|
||||
|
||||
The validator wrote every verdict in one reply and timed out five times; it now
|
||||
records them a few at a time. And two Checks named settings their resource types do
|
||||
not have, each costing a Revision; the design is now checked against the provider's
|
||||
schema, in code, before it is submitted.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from app.adk.fake_runtime import SAMPLE_DATA, sample_submission
|
||||
from app.adk.runtime import Submitted, ToolResult
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.tools.local import TerraformRunner
|
||||
from app.tools.schema_review import Reviews, SchemaReview, named_settings
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
K = DeliverableKind
|
||||
CODE = {"terraform/main.tf": "# v1\n", "implementation.json": "[]"}
|
||||
VERDICTS = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
CONTENT = sample_submission(K.VALIDATION_RESULTS).args["content"]
|
||||
|
||||
|
||||
def upstream(code=CODE) -> dict:
|
||||
return {
|
||||
"target_architecture": {
|
||||
"attachments": {
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
}
|
||||
},
|
||||
"deployment_guidance": {"attachments": dict(code)},
|
||||
}
|
||||
|
||||
|
||||
def validator_tools(working: dict, code=CODE) -> dict:
|
||||
return {
|
||||
tool.name: tool
|
||||
for tool in specialist_tools(
|
||||
K.VALIDATION_RESULTS,
|
||||
(),
|
||||
allow_cloud_read=False,
|
||||
upstream=upstream(code),
|
||||
working=working,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
# --- verdicts a few at a time ------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_verdicts_are_recorded_a_few_at_a_time_and_attached_on_submission():
|
||||
working: dict = {}
|
||||
tools = validator_tools(working)
|
||||
|
||||
first = await tools["verdict_put"].handler(
|
||||
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
|
||||
)
|
||||
second = await tools["verdict_put"].handler(checks=VERDICTS["checks"][1:])
|
||||
result = await tools["submit_deliverable"].handler(content=CONTENT)
|
||||
|
||||
assert first.message == "Recorded. 1 of 2 Checks have a verdict. Left: K-2."
|
||||
assert second.message == "Recorded. All 2 Checks have a verdict."
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
attached = json.loads(result.outcome.attachments["verdicts.json"])
|
||||
assert [item["check"] for item in attached["checks"]] == ["K-1", "K-2"]
|
||||
assert attached["dry_run"]["verdict"] == "pass"
|
||||
|
||||
|
||||
async def test_a_submission_with_a_check_left_out_names_it():
|
||||
tools = validator_tools({})
|
||||
await tools["verdict_put"].handler(
|
||||
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
|
||||
)
|
||||
|
||||
result = await tools["submit_deliverable"].handler(content=CONTENT)
|
||||
|
||||
assert result.outcome is None
|
||||
assert "Check K-2 is not accounted for" in result.message
|
||||
|
||||
|
||||
async def test_a_verdict_given_again_replaces_the_first_and_additions_are_kept():
|
||||
working: dict = {}
|
||||
tools = validator_tools(working)
|
||||
addition = {
|
||||
"statement": "A timeout is set.",
|
||||
"applies_to": "E-1",
|
||||
"reason": "NFR-1",
|
||||
}
|
||||
await tools["verdict_put"].handler(checks=VERDICTS["checks"])
|
||||
|
||||
await tools["verdict_put"].handler(
|
||||
checks=[{"check": "K-1", "verdict": "fail", "evidence": "min is 1"}],
|
||||
proposed_additions=[addition, addition],
|
||||
)
|
||||
read = await tools["verdict_read"].handler()
|
||||
|
||||
recorded = json.loads(read.message.rsplit("\n", 1)[0])
|
||||
assert [(c["check"], c["verdict"]) for c in recorded["checks"]] == [
|
||||
("K-1", "fail"),
|
||||
("K-2", "pass"),
|
||||
]
|
||||
assert recorded["proposed_additions"] == [addition]
|
||||
assert read.message.endswith("All 2 Checks have a verdict.")
|
||||
|
||||
|
||||
async def test_a_run_cut_short_carries_on_from_what_it_recorded():
|
||||
working: dict = {}
|
||||
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"][:1])
|
||||
|
||||
# The same code, a later attempt: the working memory came back with the Execution.
|
||||
read = await validator_tools(working)["verdict_read"].handler()
|
||||
|
||||
assert "1 of 2 Checks have a verdict. Left: K-2." in read.message
|
||||
|
||||
|
||||
async def test_verdicts_on_code_that_has_changed_are_dropped():
|
||||
working: dict = {}
|
||||
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"])
|
||||
|
||||
revised = {**CODE, "terraform/main.tf": "# v2\n"}
|
||||
read = await validator_tools(working, revised)["verdict_read"].handler()
|
||||
|
||||
assert read.message.startswith("No verdicts recorded yet.")
|
||||
|
||||
|
||||
async def test_verdicts_passed_as_an_attachment_still_work():
|
||||
call = sample_submission(K.VALIDATION_RESULTS)
|
||||
|
||||
result = await validator_tools({})["submit_deliverable"].handler(**call.args)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
|
||||
|
||||
def test_only_the_validator_records_verdicts():
|
||||
assert "verdict_put" in validator_tools({})
|
||||
assert "verdict_put" not in {
|
||||
tool.name
|
||||
for tool in specialist_tools(K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False)
|
||||
}
|
||||
|
||||
|
||||
# --- the design against the provider's schema --------------------------------------------
|
||||
|
||||
|
||||
def test_the_settings_a_check_names_are_read_from_its_statement():
|
||||
assert named_settings("`kms_key_name` is set on the topic") == ("kms_key_name",)
|
||||
assert named_settings("template.scaling.min_instance_count is at least 2") == (
|
||||
"template.scaling.min_instance_count",
|
||||
)
|
||||
assert named_settings("startup_cpu_boost is true") == ("startup_cpu_boost",)
|
||||
# Values, roles and plain prose name no setting.
|
||||
assert named_settings("ingress is INGRESS_TRAFFIC_ALL") == ()
|
||||
assert named_settings("roles/run.invoker is granted to allUsers") == ()
|
||||
assert named_settings('the filter contains "oldest_unacked_message_age"') == ()
|
||||
assert named_settings("The service is private.") == ()
|
||||
|
||||
|
||||
SCHEMA = {
|
||||
"cloud_service": (
|
||||
"example/cloud 1.2.3",
|
||||
{
|
||||
"attributes": {"name": {}, "ingress": {}},
|
||||
"block_types": {
|
||||
"template": {
|
||||
"block": {
|
||||
"attributes": {"service_account": {}},
|
||||
"block_types": {
|
||||
"scaling": {
|
||||
"block": {"attributes": {"min_instance_count": {}}}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
),
|
||||
"cloud_job": ("example/cloud 1.2.3", {"attributes": {"name": {}}}),
|
||||
"cloud_service_iam_member": ("example/cloud 1.2.3", {"attributes": {"role": {}}}),
|
||||
}
|
||||
|
||||
|
||||
class Runner(TerraformRunner):
|
||||
def __init__(self, schemas=SCHEMA) -> None:
|
||||
super().__init__()
|
||||
self._given = schemas
|
||||
|
||||
async def _load_schemas(self):
|
||||
return self._given
|
||||
|
||||
|
||||
def graph(checks, resource_types=("cloud_service", "cloud_job")) -> dict[str, str]:
|
||||
elements = [
|
||||
{
|
||||
"id": f"E-{n}",
|
||||
"name": f"Thing {n}",
|
||||
"product": "Example",
|
||||
"resource_type": kind,
|
||||
}
|
||||
for n, kind in enumerate(resource_types, start=1)
|
||||
]
|
||||
data = {
|
||||
"elements": elements,
|
||||
"relationships": [{"id": "R-1", "from": "E-1", "to": "E-2", "kind": "calls"}],
|
||||
"checks": [
|
||||
{"id": f"K-{n}", "applies_to": on, "statement": text, "source": "NFR-1"}
|
||||
for n, (on, text) in enumerate(checks, start=1)
|
||||
],
|
||||
}
|
||||
return {"target-architecture.json": json.dumps(data)}
|
||||
|
||||
|
||||
async def test_a_check_naming_a_setting_its_resource_type_lacks_is_raised():
|
||||
design = graph(
|
||||
[
|
||||
("E-1", "template.scaling.min_instance_count is at least 2"),
|
||||
("E-2", "`min_instance_count` is at least 2"),
|
||||
("E-2", "the name is set"),
|
||||
]
|
||||
)
|
||||
|
||||
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
|
||||
|
||||
assert len(raised) == 1
|
||||
assert raised[0].startswith(
|
||||
"Check K-2: cloud_job has no setting named min_instance_count"
|
||||
)
|
||||
|
||||
|
||||
async def test_a_setting_found_anywhere_in_the_resource_type_is_accepted():
|
||||
design = graph(
|
||||
[
|
||||
("E-1", "service_account is the dedicated identity"),
|
||||
("E-1", "ingress is INTERNAL"),
|
||||
("E-1", "a cloud_service_iam_member grants the invoker role"),
|
||||
("R-1", "any_made_up_setting is checked on a Relationship"),
|
||||
("E-2", "the name is set"),
|
||||
]
|
||||
)
|
||||
|
||||
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) == ()
|
||||
|
||||
|
||||
async def test_a_resource_type_the_provider_lacks_is_raised_with_close_matches():
|
||||
design = graph(
|
||||
[("E-1", "the name is set"), ("E-2", "the name is set")],
|
||||
resource_types=("cloud_servce", "cloud_job"),
|
||||
)
|
||||
|
||||
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
|
||||
|
||||
assert len(raised) == 1
|
||||
assert raised[0].startswith(
|
||||
"Element E-1: cloud_servce is not a resource type of the provider."
|
||||
)
|
||||
assert "Close matches: cloud_service" in raised[0]
|
||||
|
||||
|
||||
async def test_no_schema_and_other_deliverables_raise_nothing():
|
||||
design = graph([("E-1", "made_up_thing is set"), ("E-2", "the name is set")])
|
||||
unavailable = Runner(ToolResult("Schema lookup is unavailable."))
|
||||
|
||||
assert await SchemaReview(unavailable)(K.TARGET_ARCHITECTURE, design, None) == ()
|
||||
assert await SchemaReview(Runner())(K.DEPLOYMENT_GUIDANCE, design, None) == ()
|
||||
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, {}, None) == ()
|
||||
|
||||
|
||||
async def test_several_reviews_raise_together_and_one_failing_loses_nothing():
|
||||
async def one(kind, attachments, upstream, scope=""):
|
||||
return ["first"]
|
||||
|
||||
async def broken(kind, attachments, upstream, scope=""):
|
||||
raise RuntimeError("down")
|
||||
|
||||
async def two(kind, attachments, upstream, scope=""):
|
||||
return ("second",)
|
||||
|
||||
raised = await Reviews([one, broken, two])(K.TARGET_ARCHITECTURE, {}, None)
|
||||
|
||||
assert raised == ("first", "second")
|
||||
Reference in New Issue
Block a user