307 lines
10 KiB
Python
307 lines
10 KiB
Python
"""Two fixes from the run cancelled on 2026-10-07.
|
|
|
|
The validator wrote every verdict in one reply and timed out five times; it now
|
|
records them a few at a time. And two Checks named settings their resource types do
|
|
not have, each costing a Revision; the design is now checked against the provider's
|
|
schema, in code, before it is submitted.
|
|
"""
|
|
|
|
import json
|
|
|
|
from execution_engine.adk.runtime import Submitted, ToolResult
|
|
from execution_engine.tools.execution import specialist_tools
|
|
from tests.reference_workflow.local import TerraformRunner
|
|
from tests.reference_workflow.schema_review import Reviews, SchemaReview, named_settings
|
|
from tests.reference_workflow.workflow import (
|
|
SAMPLE_DATA,
|
|
WORKFLOW,
|
|
DeliverableKind,
|
|
sample_submission,
|
|
)
|
|
|
|
K = DeliverableKind
|
|
CODE = {"terraform/main.tf": "# v1\n", "implementation.json": "[]"}
|
|
VERDICTS = json.loads(SAMPLE_DATA["verdicts.json"])
|
|
CONTENT = sample_submission(K.VALIDATION_RESULTS).args["content"]
|
|
|
|
|
|
def upstream(code=CODE) -> dict:
|
|
return {
|
|
"target_architecture": {
|
|
"attachments": {
|
|
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
|
}
|
|
},
|
|
"deployment_guidance": {"attachments": dict(code)},
|
|
}
|
|
|
|
|
|
def validator_tools(working: dict, code=CODE) -> dict:
|
|
return {
|
|
tool.name: tool
|
|
for tool in specialist_tools(
|
|
WORKFLOW,
|
|
K.VALIDATION_RESULTS,
|
|
(),
|
|
allow_cloud_read=False,
|
|
upstream=upstream(code),
|
|
working=working,
|
|
)
|
|
}
|
|
|
|
|
|
# --- verdicts a few at a time ------------------------------------------------------------
|
|
|
|
|
|
async def test_verdicts_are_recorded_a_few_at_a_time_and_attached_on_submission():
|
|
working: dict = {}
|
|
tools = validator_tools(working)
|
|
|
|
first = await tools["verdict_put"].handler(
|
|
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
|
|
)
|
|
second = await tools["verdict_put"].handler(checks=VERDICTS["checks"][1:])
|
|
result = await tools["submit_deliverable"].handler(content=CONTENT)
|
|
|
|
assert first.message == "Recorded. 1 of 2 Checks have a verdict. Left: K-2."
|
|
assert second.message == "Recorded. All 2 Checks have a verdict."
|
|
assert isinstance(result.outcome, Submitted)
|
|
attached = json.loads(result.outcome.attachments["verdicts.json"])
|
|
assert [item["check"] for item in attached["checks"]] == ["K-1", "K-2"]
|
|
assert attached["dry_run"]["verdict"] == "pass"
|
|
|
|
|
|
async def test_a_submission_with_a_check_left_out_names_it():
|
|
tools = validator_tools({})
|
|
await tools["verdict_put"].handler(
|
|
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
|
|
)
|
|
|
|
result = await tools["submit_deliverable"].handler(content=CONTENT)
|
|
|
|
assert result.outcome is None
|
|
assert "Check K-2 is not accounted for" in result.message
|
|
|
|
|
|
async def test_a_verdict_given_again_replaces_the_first_and_additions_are_kept():
|
|
working: dict = {}
|
|
tools = validator_tools(working)
|
|
addition = {
|
|
"statement": "A timeout is set.",
|
|
"applies_to": "E-1",
|
|
"reason": "NFR-1",
|
|
}
|
|
await tools["verdict_put"].handler(checks=VERDICTS["checks"])
|
|
|
|
await tools["verdict_put"].handler(
|
|
checks=[{"check": "K-1", "verdict": "fail", "evidence": "min is 1"}],
|
|
proposed_additions=[addition, addition],
|
|
)
|
|
read = await tools["verdict_read"].handler()
|
|
|
|
recorded = json.loads(read.message.rsplit("\n", 1)[0])
|
|
assert [(c["check"], c["verdict"]) for c in recorded["checks"]] == [
|
|
("K-1", "fail"),
|
|
("K-2", "pass"),
|
|
]
|
|
assert recorded["proposed_additions"] == [addition]
|
|
assert read.message.endswith("All 2 Checks have a verdict.")
|
|
|
|
|
|
async def test_a_run_cut_short_carries_on_from_what_it_recorded():
|
|
working: dict = {}
|
|
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"][:1])
|
|
|
|
# The same code, a later attempt: the working memory came back with the Execution.
|
|
read = await validator_tools(working)["verdict_read"].handler()
|
|
|
|
assert "1 of 2 Checks have a verdict. Left: K-2." in read.message
|
|
|
|
|
|
async def test_verdicts_on_code_that_has_changed_are_dropped():
|
|
working: dict = {}
|
|
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"])
|
|
|
|
revised = {**CODE, "terraform/main.tf": "# v2\n"}
|
|
read = await validator_tools(working, revised)["verdict_read"].handler()
|
|
|
|
assert read.message.startswith("No verdicts recorded yet.")
|
|
|
|
|
|
async def test_verdicts_passed_as_an_attachment_still_work():
|
|
call = sample_submission(K.VALIDATION_RESULTS)
|
|
|
|
result = await validator_tools({})["submit_deliverable"].handler(**call.args)
|
|
|
|
assert isinstance(result.outcome, Submitted)
|
|
|
|
|
|
def test_only_the_validator_records_verdicts():
|
|
assert "verdict_put" in validator_tools({})
|
|
assert "verdict_put" not in {
|
|
tool.name
|
|
for tool in specialist_tools(
|
|
WORKFLOW, K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False
|
|
)
|
|
}
|
|
|
|
|
|
# --- the design against the provider's schema --------------------------------------------
|
|
|
|
|
|
def test_the_settings_a_check_names_are_read_from_its_statement():
|
|
assert named_settings("`kms_key_name` is set on the topic") == ("kms_key_name",)
|
|
assert named_settings("template.scaling.min_instance_count is at least 2") == (
|
|
"template.scaling.min_instance_count",
|
|
)
|
|
assert named_settings("startup_cpu_boost is true") == ("startup_cpu_boost",)
|
|
# A value in backticks is not a setting.
|
|
assert named_settings("`enable_cdn` is `false`") == ("enable_cdn",)
|
|
assert named_settings("`deletion_protection` is `true`") == ("deletion_protection",)
|
|
# Values, roles and plain prose name no setting.
|
|
assert named_settings("ingress is INGRESS_TRAFFIC_ALL") == ()
|
|
assert named_settings("roles/run.invoker is granted to allUsers") == ()
|
|
assert named_settings('the filter contains "oldest_unacked_message_age"') == ()
|
|
assert named_settings("The service is private.") == ()
|
|
|
|
|
|
SCHEMA = {
|
|
"cloud_service": (
|
|
"example/cloud 1.2.3",
|
|
{
|
|
"attributes": {"name": {}, "ingress": {}},
|
|
"block_types": {
|
|
"template": {
|
|
"block": {
|
|
"attributes": {"service_account": {}},
|
|
"block_types": {
|
|
"scaling": {
|
|
"block": {"attributes": {"min_instance_count": {}}}
|
|
}
|
|
},
|
|
}
|
|
}
|
|
},
|
|
},
|
|
),
|
|
"cloud_job": ("example/cloud 1.2.3", {"attributes": {"name": {}}}),
|
|
"cloud_service_iam_member": ("example/cloud 1.2.3", {"attributes": {"role": {}}}),
|
|
}
|
|
|
|
|
|
class Runner(TerraformRunner):
|
|
def __init__(self, schemas=SCHEMA) -> None:
|
|
super().__init__()
|
|
self._given = schemas
|
|
|
|
async def _load_schemas(self):
|
|
return self._given
|
|
|
|
|
|
def graph(checks, resource_types=("cloud_service", "cloud_job")) -> dict[str, str]:
|
|
elements = [
|
|
{
|
|
"id": f"E-{n}",
|
|
"name": f"Thing {n}",
|
|
"product": "Example",
|
|
"resource_type": kind,
|
|
}
|
|
for n, kind in enumerate(resource_types, start=1)
|
|
]
|
|
data = {
|
|
"elements": elements,
|
|
"relationships": [{"id": "R-1", "from": "E-1", "to": "E-2", "kind": "calls"}],
|
|
"checks": [
|
|
{"id": f"K-{n}", "applies_to": on, "statement": text, "source": "NFR-1"}
|
|
for n, (on, text) in enumerate(checks, start=1)
|
|
],
|
|
}
|
|
return {"target-architecture.json": json.dumps(data)}
|
|
|
|
|
|
async def test_a_check_naming_a_setting_its_resource_type_lacks_is_raised():
|
|
design = graph(
|
|
[
|
|
("E-1", "template.scaling.min_instance_count is at least 2"),
|
|
("E-2", "`min_instance_count` is at least 2"),
|
|
("E-2", "the name is set"),
|
|
]
|
|
)
|
|
|
|
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
|
|
|
|
assert len(raised) == 1
|
|
assert raised[0].startswith(
|
|
"Check K-2: cloud_job has no setting named min_instance_count"
|
|
)
|
|
|
|
|
|
async def test_a_setting_found_anywhere_in_the_resource_type_is_accepted():
|
|
design = graph(
|
|
[
|
|
("E-1", "service_account is the dedicated identity"),
|
|
("E-1", "ingress is INTERNAL"),
|
|
("E-1", "a cloud_service_iam_member grants the invoker role"),
|
|
("R-1", "any_made_up_setting is checked on a Relationship"),
|
|
("E-2", "the name is set"),
|
|
]
|
|
)
|
|
|
|
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) == ()
|
|
|
|
|
|
async def test_a_value_written_with_dots_is_not_read_as_a_setting():
|
|
"""A database flag or a service name is a value. A path is a setting only when
|
|
it starts at a name the resource type has."""
|
|
design = graph(
|
|
[
|
|
("E-1", "the flag cloudsql.iam_authentication is on"),
|
|
("E-1", "audit logs are on for run.googleapis.com"),
|
|
("E-1", "template.scaling.made_up_count is at least 2"),
|
|
("E-2", "the name is set"),
|
|
]
|
|
)
|
|
|
|
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
|
|
|
|
assert len(raised) == 1 and "template.scaling.made_up_count" in raised[0]
|
|
|
|
|
|
async def test_a_resource_type_the_provider_lacks_is_raised_with_close_matches():
|
|
design = graph(
|
|
[("E-1", "the name is set"), ("E-2", "the name is set")],
|
|
resource_types=("cloud_servce", "cloud_job"),
|
|
)
|
|
|
|
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
|
|
|
|
assert len(raised) == 1
|
|
assert raised[0].startswith(
|
|
"Element E-1: cloud_servce is not a resource type of the provider."
|
|
)
|
|
assert "Close matches: cloud_service" in raised[0]
|
|
|
|
|
|
async def test_no_schema_and_other_deliverables_raise_nothing():
|
|
design = graph([("E-1", "made_up_thing is set"), ("E-2", "the name is set")])
|
|
unavailable = Runner(ToolResult("Schema lookup is unavailable."))
|
|
|
|
assert await SchemaReview(unavailable)(K.TARGET_ARCHITECTURE, design, None) == ()
|
|
assert await SchemaReview(Runner())(K.DEPLOYMENT_GUIDANCE, design, None) == ()
|
|
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, {}, None) == ()
|
|
|
|
|
|
async def test_several_reviews_raise_together_and_one_failing_loses_nothing():
|
|
async def one(kind, attachments, upstream, scope=""):
|
|
return ["first"]
|
|
|
|
async def broken(kind, attachments, upstream, scope=""):
|
|
raise RuntimeError("down")
|
|
|
|
async def two(kind, attachments, upstream, scope=""):
|
|
return ("second",)
|
|
|
|
raised = await Reviews([one, broken, two])(K.TARGET_ARCHITECTURE, {}, None)
|
|
|
|
assert raised == ("first", "second")
|