feat: working orhcestration engine
This commit is contained in:
306
tests/execution/test_validation_in_pieces.py
Normal file
306
tests/execution/test_validation_in_pieces.py
Normal file
@@ -0,0 +1,306 @@
|
||||
"""Two fixes from the run cancelled on 2026-10-07.
|
||||
|
||||
The validator wrote every verdict in one reply and timed out five times; it now
|
||||
records them a few at a time. And two Checks named settings their resource types do
|
||||
not have, each costing a Revision; the design is now checked against the provider's
|
||||
schema, in code, before it is submitted.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from execution_engine.adk.runtime import Submitted, ToolResult
|
||||
from execution_engine.tools.execution import specialist_tools
|
||||
from tests.reference_workflow.local import TerraformRunner
|
||||
from tests.reference_workflow.schema_review import Reviews, SchemaReview, named_settings
|
||||
from tests.reference_workflow.workflow import (
|
||||
SAMPLE_DATA,
|
||||
WORKFLOW,
|
||||
DeliverableKind,
|
||||
sample_submission,
|
||||
)
|
||||
|
||||
K = DeliverableKind
|
||||
CODE = {"terraform/main.tf": "# v1\n", "implementation.json": "[]"}
|
||||
VERDICTS = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
CONTENT = sample_submission(K.VALIDATION_RESULTS).args["content"]
|
||||
|
||||
|
||||
def upstream(code=CODE) -> dict:
|
||||
return {
|
||||
"target_architecture": {
|
||||
"attachments": {
|
||||
"target-architecture.json": SAMPLE_DATA["target-architecture.json"]
|
||||
}
|
||||
},
|
||||
"deployment_guidance": {"attachments": dict(code)},
|
||||
}
|
||||
|
||||
|
||||
def validator_tools(working: dict, code=CODE) -> dict:
|
||||
return {
|
||||
tool.name: tool
|
||||
for tool in specialist_tools(
|
||||
WORKFLOW,
|
||||
K.VALIDATION_RESULTS,
|
||||
(),
|
||||
allow_cloud_read=False,
|
||||
upstream=upstream(code),
|
||||
working=working,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
# --- verdicts a few at a time ------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_verdicts_are_recorded_a_few_at_a_time_and_attached_on_submission():
|
||||
working: dict = {}
|
||||
tools = validator_tools(working)
|
||||
|
||||
first = await tools["verdict_put"].handler(
|
||||
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
|
||||
)
|
||||
second = await tools["verdict_put"].handler(checks=VERDICTS["checks"][1:])
|
||||
result = await tools["submit_deliverable"].handler(content=CONTENT)
|
||||
|
||||
assert first.message == "Recorded. 1 of 2 Checks have a verdict. Left: K-2."
|
||||
assert second.message == "Recorded. All 2 Checks have a verdict."
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
attached = json.loads(result.outcome.attachments["verdicts.json"])
|
||||
assert [item["check"] for item in attached["checks"]] == ["K-1", "K-2"]
|
||||
assert attached["dry_run"]["verdict"] == "pass"
|
||||
|
||||
|
||||
async def test_a_submission_with_a_check_left_out_names_it():
|
||||
tools = validator_tools({})
|
||||
await tools["verdict_put"].handler(
|
||||
dry_run=VERDICTS["dry_run"], checks=VERDICTS["checks"][:1]
|
||||
)
|
||||
|
||||
result = await tools["submit_deliverable"].handler(content=CONTENT)
|
||||
|
||||
assert result.outcome is None
|
||||
assert "Check K-2 is not accounted for" in result.message
|
||||
|
||||
|
||||
async def test_a_verdict_given_again_replaces_the_first_and_additions_are_kept():
|
||||
working: dict = {}
|
||||
tools = validator_tools(working)
|
||||
addition = {
|
||||
"statement": "A timeout is set.",
|
||||
"applies_to": "E-1",
|
||||
"reason": "NFR-1",
|
||||
}
|
||||
await tools["verdict_put"].handler(checks=VERDICTS["checks"])
|
||||
|
||||
await tools["verdict_put"].handler(
|
||||
checks=[{"check": "K-1", "verdict": "fail", "evidence": "min is 1"}],
|
||||
proposed_additions=[addition, addition],
|
||||
)
|
||||
read = await tools["verdict_read"].handler()
|
||||
|
||||
recorded = json.loads(read.message.rsplit("\n", 1)[0])
|
||||
assert [(c["check"], c["verdict"]) for c in recorded["checks"]] == [
|
||||
("K-1", "fail"),
|
||||
("K-2", "pass"),
|
||||
]
|
||||
assert recorded["proposed_additions"] == [addition]
|
||||
assert read.message.endswith("All 2 Checks have a verdict.")
|
||||
|
||||
|
||||
async def test_a_run_cut_short_carries_on_from_what_it_recorded():
|
||||
working: dict = {}
|
||||
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"][:1])
|
||||
|
||||
# The same code, a later attempt: the working memory came back with the Execution.
|
||||
read = await validator_tools(working)["verdict_read"].handler()
|
||||
|
||||
assert "1 of 2 Checks have a verdict. Left: K-2." in read.message
|
||||
|
||||
|
||||
async def test_verdicts_on_code_that_has_changed_are_dropped():
|
||||
working: dict = {}
|
||||
await validator_tools(working)["verdict_put"].handler(checks=VERDICTS["checks"])
|
||||
|
||||
revised = {**CODE, "terraform/main.tf": "# v2\n"}
|
||||
read = await validator_tools(working, revised)["verdict_read"].handler()
|
||||
|
||||
assert read.message.startswith("No verdicts recorded yet.")
|
||||
|
||||
|
||||
async def test_verdicts_passed_as_an_attachment_still_work():
|
||||
call = sample_submission(K.VALIDATION_RESULTS)
|
||||
|
||||
result = await validator_tools({})["submit_deliverable"].handler(**call.args)
|
||||
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
|
||||
|
||||
def test_only_the_validator_records_verdicts():
|
||||
assert "verdict_put" in validator_tools({})
|
||||
assert "verdict_put" not in {
|
||||
tool.name
|
||||
for tool in specialist_tools(
|
||||
WORKFLOW, K.DEPLOYMENT_GUIDANCE, (), allow_cloud_read=False
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
# --- the design against the provider's schema --------------------------------------------
|
||||
|
||||
|
||||
def test_the_settings_a_check_names_are_read_from_its_statement():
|
||||
assert named_settings("`kms_key_name` is set on the topic") == ("kms_key_name",)
|
||||
assert named_settings("template.scaling.min_instance_count is at least 2") == (
|
||||
"template.scaling.min_instance_count",
|
||||
)
|
||||
assert named_settings("startup_cpu_boost is true") == ("startup_cpu_boost",)
|
||||
# A value in backticks is not a setting.
|
||||
assert named_settings("`enable_cdn` is `false`") == ("enable_cdn",)
|
||||
assert named_settings("`deletion_protection` is `true`") == ("deletion_protection",)
|
||||
# Values, roles and plain prose name no setting.
|
||||
assert named_settings("ingress is INGRESS_TRAFFIC_ALL") == ()
|
||||
assert named_settings("roles/run.invoker is granted to allUsers") == ()
|
||||
assert named_settings('the filter contains "oldest_unacked_message_age"') == ()
|
||||
assert named_settings("The service is private.") == ()
|
||||
|
||||
|
||||
SCHEMA = {
|
||||
"cloud_service": (
|
||||
"example/cloud 1.2.3",
|
||||
{
|
||||
"attributes": {"name": {}, "ingress": {}},
|
||||
"block_types": {
|
||||
"template": {
|
||||
"block": {
|
||||
"attributes": {"service_account": {}},
|
||||
"block_types": {
|
||||
"scaling": {
|
||||
"block": {"attributes": {"min_instance_count": {}}}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
),
|
||||
"cloud_job": ("example/cloud 1.2.3", {"attributes": {"name": {}}}),
|
||||
"cloud_service_iam_member": ("example/cloud 1.2.3", {"attributes": {"role": {}}}),
|
||||
}
|
||||
|
||||
|
||||
class Runner(TerraformRunner):
|
||||
def __init__(self, schemas=SCHEMA) -> None:
|
||||
super().__init__()
|
||||
self._given = schemas
|
||||
|
||||
async def _load_schemas(self):
|
||||
return self._given
|
||||
|
||||
|
||||
def graph(checks, resource_types=("cloud_service", "cloud_job")) -> dict[str, str]:
|
||||
elements = [
|
||||
{
|
||||
"id": f"E-{n}",
|
||||
"name": f"Thing {n}",
|
||||
"product": "Example",
|
||||
"resource_type": kind,
|
||||
}
|
||||
for n, kind in enumerate(resource_types, start=1)
|
||||
]
|
||||
data = {
|
||||
"elements": elements,
|
||||
"relationships": [{"id": "R-1", "from": "E-1", "to": "E-2", "kind": "calls"}],
|
||||
"checks": [
|
||||
{"id": f"K-{n}", "applies_to": on, "statement": text, "source": "NFR-1"}
|
||||
for n, (on, text) in enumerate(checks, start=1)
|
||||
],
|
||||
}
|
||||
return {"target-architecture.json": json.dumps(data)}
|
||||
|
||||
|
||||
async def test_a_check_naming_a_setting_its_resource_type_lacks_is_raised():
|
||||
design = graph(
|
||||
[
|
||||
("E-1", "template.scaling.min_instance_count is at least 2"),
|
||||
("E-2", "`min_instance_count` is at least 2"),
|
||||
("E-2", "the name is set"),
|
||||
]
|
||||
)
|
||||
|
||||
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
|
||||
|
||||
assert len(raised) == 1
|
||||
assert raised[0].startswith(
|
||||
"Check K-2: cloud_job has no setting named min_instance_count"
|
||||
)
|
||||
|
||||
|
||||
async def test_a_setting_found_anywhere_in_the_resource_type_is_accepted():
|
||||
design = graph(
|
||||
[
|
||||
("E-1", "service_account is the dedicated identity"),
|
||||
("E-1", "ingress is INTERNAL"),
|
||||
("E-1", "a cloud_service_iam_member grants the invoker role"),
|
||||
("R-1", "any_made_up_setting is checked on a Relationship"),
|
||||
("E-2", "the name is set"),
|
||||
]
|
||||
)
|
||||
|
||||
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) == ()
|
||||
|
||||
|
||||
async def test_a_value_written_with_dots_is_not_read_as_a_setting():
|
||||
"""A database flag or a service name is a value. A path is a setting only when
|
||||
it starts at a name the resource type has."""
|
||||
design = graph(
|
||||
[
|
||||
("E-1", "the flag cloudsql.iam_authentication is on"),
|
||||
("E-1", "audit logs are on for run.googleapis.com"),
|
||||
("E-1", "template.scaling.made_up_count is at least 2"),
|
||||
("E-2", "the name is set"),
|
||||
]
|
||||
)
|
||||
|
||||
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
|
||||
|
||||
assert len(raised) == 1 and "template.scaling.made_up_count" in raised[0]
|
||||
|
||||
|
||||
async def test_a_resource_type_the_provider_lacks_is_raised_with_close_matches():
|
||||
design = graph(
|
||||
[("E-1", "the name is set"), ("E-2", "the name is set")],
|
||||
resource_types=("cloud_servce", "cloud_job"),
|
||||
)
|
||||
|
||||
raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None)
|
||||
|
||||
assert len(raised) == 1
|
||||
assert raised[0].startswith(
|
||||
"Element E-1: cloud_servce is not a resource type of the provider."
|
||||
)
|
||||
assert "Close matches: cloud_service" in raised[0]
|
||||
|
||||
|
||||
async def test_no_schema_and_other_deliverables_raise_nothing():
|
||||
design = graph([("E-1", "made_up_thing is set"), ("E-2", "the name is set")])
|
||||
unavailable = Runner(ToolResult("Schema lookup is unavailable."))
|
||||
|
||||
assert await SchemaReview(unavailable)(K.TARGET_ARCHITECTURE, design, None) == ()
|
||||
assert await SchemaReview(Runner())(K.DEPLOYMENT_GUIDANCE, design, None) == ()
|
||||
assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, {}, None) == ()
|
||||
|
||||
|
||||
async def test_several_reviews_raise_together_and_one_failing_loses_nothing():
|
||||
async def one(kind, attachments, upstream, scope=""):
|
||||
return ["first"]
|
||||
|
||||
async def broken(kind, attachments, upstream, scope=""):
|
||||
raise RuntimeError("down")
|
||||
|
||||
async def two(kind, attachments, upstream, scope=""):
|
||||
return ("second",)
|
||||
|
||||
raised = await Reviews([one, broken, two])(K.TARGET_ARCHITECTURE, {}, None)
|
||||
|
||||
assert raised == ("first", "second")
|
||||
Reference in New Issue
Block a user