Files
orchestration_engine/tests/execution/test_validation_in_pieces.py

307 lines
10 KiB
Python

"""Two fixes from the run cancelled on 2026-10-07.
The validator wrote every verdict in one reply and timed out five times; it now
records them a few at a time. And two Checks named settings their resource types do
not have, each costing a Revision; the design is now checked against the provider's
schema, in code, before it is submitted.
"""
import json
from execution_engine.adk.runtime import Submitted, ToolResult
from execution_engine.tools.execution import specialist_tools
from tests.reference_workflow.local import TerraformRunner
from tests.reference_workflow.schema_review import Reviews, SchemaReview, named_settings
from tests.reference_workflow.workflow import (
SAMPLE_DATA,
WORKFLOW,
DeliverableKind,
sample_submission,
)
K = DeliverableKind
CODE = {"terraform/main.tf": "# v1\n", "implementation.json": "[]"}
VERDICTS = json.loads(SAMPLE_DATA["verdicts.json"])
CONTENT = sample_submission(K.VALIDATION_RESULTS).args["content"]
def upstream(code=CODE) -> dict:
return {
"target_architecture": {
"attachments": {
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
}
},
"deployment_guidance": {"attachments": dict(code)},
}
def validator_tools(working: dict, code=CODE) -> dict:
return {
tool.name: tool
for tool in specialist_tools(
WORKFLOW,
K.VALIDATION_RESULTS,
(),
allow_cloud_read=False,
upstream=upstream(code),
working=working,
)
}
# --- verdicts a few at a time ------------------------------------------------------------
async def test_verdicts_are_recorded_a_few_at_a_time_and_attached_on_submission():
working: dict = {}
tools = validator_tools(working)
first = await tools["verdict_put"].handler(
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
)
second = await tools["verdict_put"].handler(checks=VERDICTS["checks"][1:])
result = await tools["submit_deliverable"].handler(content=CONTENT)
assert first.message == "Recorded. 1 of 2 Checks have a verdict. Left: K-2."
assert second.message == "Recorded. All 2 Checks have a verdict."
assert isinstance(result.outcome, Submitted)
attached = json.loads(result.outcome.attachments["verdicts.json"])
assert [item["check"] for item in attached["checks"]] == ["K-1", "K-2"]
assert attached["dry_run"]["verdict"] == "pass"
async def test_a_submission_with_a_check_left_out_names_it():
tools = validator_tools({})
await tools["verdict_put"].handler(
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
)
result = await tools["submit_deliverable"].handler(content=CONTENT)
assert result.outcome is None
assert "Check K-2 is not accounted for" in result.message
async def test_a_verdict_given_again_replaces_the_first_and_additions_are_kept():
working: dict = {}
tools = validator_tools(working)
addition = {
"statement": "A timeout is set.",
"applies_to": "E-1",
"reason": "NFR-1",
}
await tools["verdict_put"].handler(checks=VERDICTS["checks"])
await tools["verdict_put"].handler(
checks=[{"check": "K-1", "verdict": "fail", "evidence": "min is 1"}],
proposed_additions=[addition, addition],
)
read = await tools["verdict_read"].handler()
recorded = json.loads(read.message.rsplit("\n", 1)[0])
assert [(c["check"], c["verdict"]) for c in recorded["checks"]] == [
("K-1", "fail"),
("K-2", "pass"),
]
assert recorded["proposed_additions"] == [addition]
assert read.message.endswith("All 2 Checks have a verdict.")
async def test_a_run_cut_short_carries_on_from_what_it_recorded():
working: dict = {}
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"][:1])
# The same code, a later attempt: the working memory came back with the Execution.
read = await validator_tools(working)["verdict_read"].handler()
assert "1 of 2 Checks have a verdict. Left: K-2." in read.message
async def test_verdicts_on_code_that_has_changed_are_dropped():
working: dict = {}
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"])
revised = {**CODE, "terraform/main.tf": "# v2\n"}
read = await validator_tools(working, revised)["verdict_read"].handler()
assert read.message.startswith("No verdicts recorded yet.")
async def test_verdicts_passed_as_an_attachment_still_work():
call = sample_submission(K.VALIDATION_RESULTS)
result = await validator_tools({})["submit_deliverable"].handler(**call.args)
assert isinstance(result.outcome, Submitted)
def test_only_the_validator_records_verdicts():
assert "verdict_put" in validator_tools({})
assert "verdict_put" not in {
tool.name
for tool in specialist_tools(
WORKFLOW, K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False
)
}
# --- the design against the provider's schema --------------------------------------------
def test_the_settings_a_check_names_are_read_from_its_statement():
assert named_settings("`kms_key_name` is set on the topic") == ("kms_key_name",)
assert named_settings("template.scaling.min_instance_count is at least 2") == (
"template.scaling.min_instance_count",
)
assert named_settings("startup_cpu_boost is true") == ("startup_cpu_boost",)
# A value in backticks is not a setting.
assert named_settings("`enable_cdn` is `false`") == ("enable_cdn",)
assert named_settings("`deletion_protection` is `true`") == ("deletion_protection",)
# Values, roles and plain prose name no setting.
assert named_settings("ingress is INGRESS_TRAFFIC_ALL") == ()
assert named_settings("roles/run.invoker is granted to allUsers") == ()
assert named_settings('the filter contains "oldest_unacked_message_age"') == ()
assert named_settings("The service is private.") == ()
SCHEMA = {
"cloud_service": (
"example/cloud 1.2.3",
{
"attributes": {"name": {}, "ingress": {}},
"block_types": {
"template": {
"block": {
"attributes": {"service_account": {}},
"block_types": {
"scaling": {
"block": {"attributes": {"min_instance_count": {}}}
}
},
}
}
},
},
),
"cloud_job": ("example/cloud 1.2.3", {"attributes": {"name": {}}}),
"cloud_service_iam_member": ("example/cloud 1.2.3", {"attributes": {"role": {}}}),
}
class Runner(TerraformRunner):
def __init__(self, schemas=SCHEMA) -> None:
super().__init__()
self._given = schemas
async def _load_schemas(self):
return self._given
def graph(checks, resource_types=("cloud_service", "cloud_job")) -> dict[str, str]:
elements = [
{
"id": f"E-{n}",
"name": f"Thing {n}",
"product": "Example",
"resource_type": kind,
}
for n, kind in enumerate(resource_types, start=1)
]
data = {
"elements": elements,
"relationships": [{"id": "R-1", "from": "E-1", "to": "E-2", "kind": "calls"}],
"checks": [
{"id": f"K-{n}", "applies_to": on, "statement": text, "source": "NFR-1"}
for n, (on, text) in enumerate(checks, start=1)
],
}
return {"target-architecture.json": json.dumps(data)}
async def test_a_check_naming_a_setting_its_resource_type_lacks_is_raised():
design = graph(
[
("E-1", "template.scaling.min_instance_count is at least 2"),
("E-2", "`min_instance_count` is at least 2"),
("E-2", "the name is set"),
]
)
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
assert len(raised) == 1
assert raised[0].startswith(
"Check K-2: cloud_job has no setting named min_instance_count"
)
async def test_a_setting_found_anywhere_in_the_resource_type_is_accepted():
design = graph(
[
("E-1", "service_account is the dedicated identity"),
("E-1", "ingress is INTERNAL"),
("E-1", "a cloud_service_iam_member grants the invoker role"),
("R-1", "any_made_up_setting is checked on a Relationship"),
("E-2", "the name is set"),
]
)
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) == ()
async def test_a_value_written_with_dots_is_not_read_as_a_setting():
"""A database flag or a service name is a value. A path is a setting only when
it starts at a name the resource type has."""
design = graph(
[
("E-1", "the flag cloudsql.iam_authentication is on"),
("E-1", "audit logs are on for run.googleapis.com"),
("E-1", "template.scaling.made_up_count is at least 2"),
("E-2", "the name is set"),
]
)
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
assert len(raised) == 1 and "template.scaling.made_up_count" in raised[0]
async def test_a_resource_type_the_provider_lacks_is_raised_with_close_matches():
design = graph(
[("E-1", "the name is set"), ("E-2", "the name is set")],
resource_types=("cloud_servce", "cloud_job"),
)
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
assert len(raised) == 1
assert raised[0].startswith(
"Element E-1: cloud_servce is not a resource type of the provider."
)
assert "Close matches: cloud_service" in raised[0]
async def test_no_schema_and_other_deliverables_raise_nothing():
design = graph([("E-1", "made_up_thing is set"), ("E-2", "the name is set")])
unavailable = Runner(ToolResult("Schema lookup is unavailable."))
assert await SchemaReview(unavailable)(K.TARGET_ARCHITECTURE, design, None) == ()
assert await SchemaReview(Runner())(K.DEPLOYMENT_GUIDANCE, design, None) == ()
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, {}, None) == ()
async def test_several_reviews_raise_together_and_one_failing_loses_nothing():
async def one(kind, attachments, upstream, scope=""):
return ["first"]
async def broken(kind, attachments, upstream, scope=""):
raise RuntimeError("down")
async def two(kind, attachments, upstream, scope=""):
return ("second",)
raised = await Reviews([one, broken, two])(K.TARGET_ARCHITECTURE, {}, None)
assert raised == ("first", "second")