"""Two fixes from the run cancelled on 2026-10-07. The validator wrote every verdict in one reply and timed out five times; it now records them a few at a time. And two Checks named settings their resource types do not have, each costing a Revision; the design is now checked against the provider's schema, in code, before it is submitted. """ import json from execution_engine.adk.runtime import Submitted, ToolResult from execution_engine.tools.execution import specialist_tools from tests.reference_workflow.local import TerraformRunner from tests.reference_workflow.schema_review import Reviews, SchemaReview, named_settings from tests.reference_workflow.workflow import ( SAMPLE_DATA, WORKFLOW, DeliverableKind, sample_submission, ) K = DeliverableKind CODE = {"terraform/main.tf": "# v1\n", "implementation.json": "[]"} VERDICTS = json.loads(SAMPLE_DATA["verdicts.json"]) CONTENT = sample_submission(K.VALIDATION_RESULTS).args["content"] def upstream(code=CODE) -> dict: return { "target_architecture": { "attachments": { "target-architecture.json": SAMPLE_DATA["target-architecture.json"] } }, "deployment_guidance": {"attachments": dict(code)}, } def validator_tools(working: dict, code=CODE) -> dict: return { tool.name: tool for tool in specialist_tools( WORKFLOW, K.VALIDATION_RESULTS, (), allow_cloud_read=False, upstream=upstream(code), working=working, ) } # --- verdicts a few at a time ------------------------------------------------------------ async def test_verdicts_are_recorded_a_few_at_a_time_and_attached_on_submission(): working: dict = {} tools = validator_tools(working) first = await tools["verdict_put"].handler( dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1] ) second = await tools["verdict_put"].handler(checks=VERDICTS["checks"][1:]) result = await tools["submit_deliverable"].handler(content=CONTENT) assert first.message == "Recorded. 1 of 2 Checks have a verdict. Left: K-2." assert second.message == "Recorded. All 2 Checks have a verdict." assert isinstance(result.outcome, Submitted) attached = json.loads(result.outcome.attachments["verdicts.json"]) assert [item["check"] for item in attached["checks"]] == ["K-1", "K-2"] assert attached["dry_run"]["verdict"] == "pass" async def test_a_submission_with_a_check_left_out_names_it(): tools = validator_tools({}) await tools["verdict_put"].handler( dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1] ) result = await tools["submit_deliverable"].handler(content=CONTENT) assert result.outcome is None assert "Check K-2 is not accounted for" in result.message async def test_a_verdict_given_again_replaces_the_first_and_additions_are_kept(): working: dict = {} tools = validator_tools(working) addition = { "statement": "A timeout is set.", "applies_to": "E-1", "reason": "NFR-1", } await tools["verdict_put"].handler(checks=VERDICTS["checks"]) await tools["verdict_put"].handler( checks=[{"check": "K-1", "verdict": "fail", "evidence": "min is 1"}], proposed_additions=[addition, addition], ) read = await tools["verdict_read"].handler() recorded = json.loads(read.message.rsplit("\n", 1)[0]) assert [(c["check"], c["verdict"]) for c in recorded["checks"]] == [ ("K-1", "fail"), ("K-2", "pass"), ] assert recorded["proposed_additions"] == [addition] assert read.message.endswith("All 2 Checks have a verdict.") async def test_a_run_cut_short_carries_on_from_what_it_recorded(): working: dict = {} await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"][:1]) # The same code, a later attempt: the working memory came back with the Execution. read = await validator_tools(working)["verdict_read"].handler() assert "1 of 2 Checks have a verdict. Left: K-2." in read.message async def test_verdicts_on_code_that_has_changed_are_dropped(): working: dict = {} await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"]) revised = {**CODE, "terraform/main.tf": "# v2\n"} read = await validator_tools(working, revised)["verdict_read"].handler() assert read.message.startswith("No verdicts recorded yet.") async def test_verdicts_passed_as_an_attachment_still_work(): call = sample_submission(K.VALIDATION_RESULTS) result = await validator_tools({})["submit_deliverable"].handler(**call.args) assert isinstance(result.outcome, Submitted) def test_only_the_validator_records_verdicts(): assert "verdict_put" in validator_tools({}) assert "verdict_put" not in { tool.name for tool in specialist_tools( WORKFLOW, K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False ) } # --- the design against the provider's schema -------------------------------------------- def test_the_settings_a_check_names_are_read_from_its_statement(): assert named_settings("`kms_key_name` is set on the topic") == ("kms_key_name",) assert named_settings("template.scaling.min_instance_count is at least 2") == ( "template.scaling.min_instance_count", ) assert named_settings("startup_cpu_boost is true") == ("startup_cpu_boost",) # A value in backticks is not a setting. assert named_settings("`enable_cdn` is `false`") == ("enable_cdn",) assert named_settings("`deletion_protection` is `true`") == ("deletion_protection",) # Values, roles and plain prose name no setting. assert named_settings("ingress is INGRESS_TRAFFIC_ALL") == () assert named_settings("roles/run.invoker is granted to allUsers") == () assert named_settings('the filter contains "oldest_unacked_message_age"') == () assert named_settings("The service is private.") == () SCHEMA = { "cloud_service": ( "example/cloud 1.2.3", { "attributes": {"name": {}, "ingress": {}}, "block_types": { "template": { "block": { "attributes": {"service_account": {}}, "block_types": { "scaling": { "block": {"attributes": {"min_instance_count": {}}} } }, } } }, }, ), "cloud_job": ("example/cloud 1.2.3", {"attributes": {"name": {}}}), "cloud_service_iam_member": ("example/cloud 1.2.3", {"attributes": {"role": {}}}), } class Runner(TerraformRunner): def __init__(self, schemas=SCHEMA) -> None: super().__init__() self._given = schemas async def _load_schemas(self): return self._given def graph(checks, resource_types=("cloud_service", "cloud_job")) -> dict[str, str]: elements = [ { "id": f"E-{n}", "name": f"Thing {n}", "product": "Example", "resource_type": kind, } for n, kind in enumerate(resource_types, start=1) ] data = { "elements": elements, "relationships": [{"id": "R-1", "from": "E-1", "to": "E-2", "kind": "calls"}], "checks": [ {"id": f"K-{n}", "applies_to": on, "statement": text, "source": "NFR-1"} for n, (on, text) in enumerate(checks, start=1) ], } return {"target-architecture.json": json.dumps(data)} async def test_a_check_naming_a_setting_its_resource_type_lacks_is_raised(): design = graph( [ ("E-1", "template.scaling.min_instance_count is at least 2"), ("E-2", "`min_instance_count` is at least 2"), ("E-2", "the name is set"), ] ) raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) assert len(raised) == 1 assert raised[0].startswith( "Check K-2: cloud_job has no setting named min_instance_count" ) async def test_a_setting_found_anywhere_in_the_resource_type_is_accepted(): design = graph( [ ("E-1", "service_account is the dedicated identity"), ("E-1", "ingress is INTERNAL"), ("E-1", "a cloud_service_iam_member grants the invoker role"), ("R-1", "any_made_up_setting is checked on a Relationship"), ("E-2", "the name is set"), ] ) assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) == () async def test_a_value_written_with_dots_is_not_read_as_a_setting(): """A database flag or a service name is a value. A path is a setting only when it starts at a name the resource type has.""" design = graph( [ ("E-1", "the flag cloudsql.iam_authentication is on"), ("E-1", "audit logs are on for run.googleapis.com"), ("E-1", "template.scaling.made_up_count is at least 2"), ("E-2", "the name is set"), ] ) raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) assert len(raised) == 1 and "template.scaling.made_up_count" in raised[0] async def test_a_resource_type_the_provider_lacks_is_raised_with_close_matches(): design = graph( [("E-1", "the name is set"), ("E-2", "the name is set")], resource_types=("cloud_servce", "cloud_job"), ) raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) assert len(raised) == 1 assert raised[0].startswith( "Element E-1: cloud_servce is not a resource type of the provider." ) assert "Close matches: cloud_service" in raised[0] async def test_no_schema_and_other_deliverables_raise_nothing(): design = graph([("E-1", "made_up_thing is set"), ("E-2", "the name is set")]) unavailable = Runner(ToolResult("Schema lookup is unavailable.")) assert await SchemaReview(unavailable)(K.TARGET_ARCHITECTURE, design, None) == () assert await SchemaReview(Runner())(K.DEPLOYMENT_GUIDANCE, design, None) == () assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, {}, None) == () async def test_several_reviews_raise_together_and_one_failing_loses_nothing(): async def one(kind, attachments, upstream, scope=""): return ["first"] async def broken(kind, attachments, upstream, scope=""): raise RuntimeError("down") async def two(kind, attachments, upstream, scope=""): return ("second",) raised = await Reviews([one, broken, two])(K.TARGET_ARCHITECTURE, {}, None) assert raised == ("first", "second")