latest refactoring
Some checks failed
validation / verify (push) Failing after 10s

This commit is contained in:
2026-10-06 20:22:37 +01:00
parent e94b2493e7
commit 7ee49ab352
414 changed files with 15768 additions and 3896 deletions

0
tests/eval/__init__.py Normal file
View File

183
tests/eval/test_scoring.py Normal file
View File

@@ -0,0 +1,183 @@
"""Scoring a finished Execution against a benchmark case, without models or network."""
from pathlib import Path
from app.adk.runtime import ToolResult
from app.agent import Interaction
from eval.scoring import CaseSpec, score_execution, template_headings
TEMPLATE = (
Path(__file__).parents[2]
/ "skills/vendor/google/google-cloud-solution-architecture/assets/output-template.md"
)
HEADINGS = template_headings(TEMPLATE)
CASE = CaseSpec(
id="case-x",
name="Webhook ingestion",
workflow_request="Ingest signed webhooks durably and keep 30 days of raw events.",
required_products=("Cloud Run", "Pub/Sub", "Cloud Storage"),
scripted_answers=(),
)
def guide(headings=HEADINGS) -> str:
return "\n".join(f"## {heading}\nContent." for heading in headings)
def execution(status="completed", **overrides) -> Interaction:
deliverables = {
"requirements": {
"content": "## Functional requirements\nAccept webhooks.\n## Current state\nNone.",
"attachments": {},
},
"product_selection": {
"content": (
"## Products and features mapping\n"
"| Component | Product | Justification | Alternatives | Pros and cons |\n"
"| Ingress | Cloud Run | [docs](https://cloud.google.com/run/docs) | GKE | "
"**Pros**: control **Cons**: toil |\n"
"| Queue | Pub/Sub | ... | Cloud Tasks | ... |\n| Archive | Cloud Storage | ... |"
),
"attachments": {},
},
"architecture_diagram": {
"content": "diagram",
"attachments": {
"architecture.mmd": "flowchart TD\n A[Client] --> B[Run]\n"
},
},
"deployment_guidance": {
"content": "guide",
"attachments": {
"terraform/main.tf": "resource {}",
"terraform/variables.tf": "",
},
},
"solution_guide": {"content": guide(), "attachments": {}},
}
for kind, value in overrides.items():
deliverables[kind] = value
return Interaction(
"exec-1", status, None, {"deliverables": deliverables, "warnings": []}
)
async def passing_terraform(files) -> ToolResult:
return ToolResult("terraform validate passed:\nSuccess!")
async def failing_terraform(files) -> ToolResult:
return ToolResult("terraform validate failed:\nError: Unsupported argument")
def check(result, name):
return next(c for c in result.checks if c.name == name)
def test_template_headings_skip_the_title_and_placeholders():
assert HEADINGS[0] == "1. Executive summary and workload overview"
assert "4.1. Google Cloud products and features mapping" in HEADINGS
assert not any("[" in heading for heading in HEADINGS)
assert len(HEADINGS) >= 20
async def test_a_complete_execution_passes_every_check():
result = await score_execution(CASE, execution(), HEADINGS, passing_terraform)
assert result.passed, [c for c in result.checks if not c.passed]
assert result.score == 1.0
async def test_missing_template_sections_are_named():
partial = {"content": guide(HEADINGS[:-2]), "attachments": {}}
result = await score_execution(
CASE, execution(solution_guide=partial), HEADINGS, passing_terraform
)
detail = check(result, "guide_follows_template").detail
assert not check(result, "guide_follows_template").passed
assert HEADINGS[-1] in detail
async def test_requirements_naming_products_not_in_the_request_breaks_phase_separation():
leaky = {"content": "## Functional requirements\nUse Cloud Run.", "attachments": {}}
result = await score_execution(
CASE, execution(requirements=leaky), HEADINGS, passing_terraform
)
assert not check(result, "requirements_name_no_products").passed
assert "Cloud Run" in check(result, "requirements_name_no_products").detail
async def test_products_named_by_the_user_may_appear_in_requirements():
case = CaseSpec(
**{**CASE.__dict__, "workflow_request": "Use Cloud Run for webhooks."}
)
named = {
"content": "## Functional requirements\nRuns on Cloud Run.",
"attachments": {},
}
result = await score_execution(
case, execution(requirements=named), HEADINGS, passing_terraform
)
assert check(result, "requirements_name_no_products").passed
async def test_selection_without_alternatives_or_citations_fails_those_checks():
bare = {
"content": "## Products and features mapping\nCloud Run, Pub/Sub, Cloud Storage.",
"attachments": {},
}
result = await score_execution(
CASE, execution(product_selection=bare), HEADINGS, passing_terraform
)
assert not check(result, "selection_has_alternatives").passed
assert not check(result, "selection_cites_documentation").passed
assert check(result, "required_products_selected").passed
async def test_invalid_terraform_and_diagram_fail_and_warnings_fail_completion():
diagram = {"content": "d", "attachments": {"architecture.mmd": "A --> B"}}
result = await score_execution(
CASE,
execution("completed_with_warnings", architecture_diagram=diagram),
HEADINGS,
failing_terraform,
)
assert not check(result, "completed").passed
assert not check(result, "terraform_validates").passed
assert not check(result, "diagram_is_valid_mermaid").passed
assert not result.passed
async def test_an_execution_that_never_produced_deliverables_scores_zero_safely():
empty = Interaction("exec-2", "completed_with_warnings", None, {"deliverables": {}})
result = await score_execution(CASE, empty, HEADINGS, passing_terraform)
assert result.score == 0.0
assert not result.passed
def test_benchmark_cases_load_with_valid_answer_patterns():
import re
from eval.scoring import load_cases
cases = load_cases(Path(__file__).parents[2] / "eval/datasets/solution_cases.json")
assert len(cases) >= 2
for case in cases:
assert case.required_products
for pattern, answer in case.scripted_answers:
re.compile(pattern)
assert answer