This commit is contained in:
0
tests/eval/__init__.py
Normal file
0
tests/eval/__init__.py
Normal file
183
tests/eval/test_scoring.py
Normal file
183
tests/eval/test_scoring.py
Normal file
@@ -0,0 +1,183 @@
|
||||
"""Scoring a finished Execution against a benchmark case, without models or network."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.adk.runtime import ToolResult
|
||||
from app.agent import Interaction
|
||||
from eval.scoring import CaseSpec, score_execution, template_headings
|
||||
|
||||
TEMPLATE = (
|
||||
Path(__file__).parents[2]
|
||||
/ "skills/vendor/google/google-cloud-solution-architecture/assets/output-template.md"
|
||||
)
|
||||
HEADINGS = template_headings(TEMPLATE)
|
||||
|
||||
CASE = CaseSpec(
|
||||
id="case-x",
|
||||
name="Webhook ingestion",
|
||||
workflow_request="Ingest signed webhooks durably and keep 30 days of raw events.",
|
||||
required_products=("Cloud Run", "Pub/Sub", "Cloud Storage"),
|
||||
scripted_answers=(),
|
||||
)
|
||||
|
||||
|
||||
def guide(headings=HEADINGS) -> str:
|
||||
return "\n".join(f"## {heading}\nContent." for heading in headings)
|
||||
|
||||
|
||||
def execution(status="completed", **overrides) -> Interaction:
|
||||
deliverables = {
|
||||
"requirements": {
|
||||
"content": "## Functional requirements\nAccept webhooks.\n## Current state\nNone.",
|
||||
"attachments": {},
|
||||
},
|
||||
"product_selection": {
|
||||
"content": (
|
||||
"## Products and features mapping\n"
|
||||
"| Component | Product | Justification | Alternatives | Pros and cons |\n"
|
||||
"| Ingress | Cloud Run | [docs](https://cloud.google.com/run/docs) | GKE | "
|
||||
"**Pros**: control **Cons**: toil |\n"
|
||||
"| Queue | Pub/Sub | ... | Cloud Tasks | ... |\n| Archive | Cloud Storage | ... |"
|
||||
),
|
||||
"attachments": {},
|
||||
},
|
||||
"architecture_diagram": {
|
||||
"content": "diagram",
|
||||
"attachments": {
|
||||
"architecture.mmd": "flowchart TD\n A[Client] --> B[Run]\n"
|
||||
},
|
||||
},
|
||||
"deployment_guidance": {
|
||||
"content": "guide",
|
||||
"attachments": {
|
||||
"terraform/main.tf": "resource {}",
|
||||
"terraform/variables.tf": "",
|
||||
},
|
||||
},
|
||||
"solution_guide": {"content": guide(), "attachments": {}},
|
||||
}
|
||||
for kind, value in overrides.items():
|
||||
deliverables[kind] = value
|
||||
return Interaction(
|
||||
"exec-1", status, None, {"deliverables": deliverables, "warnings": []}
|
||||
)
|
||||
|
||||
|
||||
async def passing_terraform(files) -> ToolResult:
|
||||
return ToolResult("terraform validate passed:\nSuccess!")
|
||||
|
||||
|
||||
async def failing_terraform(files) -> ToolResult:
|
||||
return ToolResult("terraform validate failed:\nError: Unsupported argument")
|
||||
|
||||
|
||||
def check(result, name):
|
||||
return next(c for c in result.checks if c.name == name)
|
||||
|
||||
|
||||
def test_template_headings_skip_the_title_and_placeholders():
|
||||
assert HEADINGS[0] == "1. Executive summary and workload overview"
|
||||
assert "4.1. Google Cloud products and features mapping" in HEADINGS
|
||||
assert not any("[" in heading for heading in HEADINGS)
|
||||
assert len(HEADINGS) >= 20
|
||||
|
||||
|
||||
async def test_a_complete_execution_passes_every_check():
|
||||
result = await score_execution(CASE, execution(), HEADINGS, passing_terraform)
|
||||
|
||||
assert result.passed, [c for c in result.checks if not c.passed]
|
||||
assert result.score == 1.0
|
||||
|
||||
|
||||
async def test_missing_template_sections_are_named():
|
||||
partial = {"content": guide(HEADINGS[:-2]), "attachments": {}}
|
||||
|
||||
result = await score_execution(
|
||||
CASE, execution(solution_guide=partial), HEADINGS, passing_terraform
|
||||
)
|
||||
|
||||
detail = check(result, "guide_follows_template").detail
|
||||
assert not check(result, "guide_follows_template").passed
|
||||
assert HEADINGS[-1] in detail
|
||||
|
||||
|
||||
async def test_requirements_naming_products_not_in_the_request_breaks_phase_separation():
|
||||
leaky = {"content": "## Functional requirements\nUse Cloud Run.", "attachments": {}}
|
||||
|
||||
result = await score_execution(
|
||||
CASE, execution(requirements=leaky), HEADINGS, passing_terraform
|
||||
)
|
||||
|
||||
assert not check(result, "requirements_name_no_products").passed
|
||||
assert "Cloud Run" in check(result, "requirements_name_no_products").detail
|
||||
|
||||
|
||||
async def test_products_named_by_the_user_may_appear_in_requirements():
|
||||
case = CaseSpec(
|
||||
**{**CASE.__dict__, "workflow_request": "Use Cloud Run for webhooks."}
|
||||
)
|
||||
named = {
|
||||
"content": "## Functional requirements\nRuns on Cloud Run.",
|
||||
"attachments": {},
|
||||
}
|
||||
|
||||
result = await score_execution(
|
||||
case, execution(requirements=named), HEADINGS, passing_terraform
|
||||
)
|
||||
|
||||
assert check(result, "requirements_name_no_products").passed
|
||||
|
||||
|
||||
async def test_selection_without_alternatives_or_citations_fails_those_checks():
|
||||
bare = {
|
||||
"content": "## Products and features mapping\nCloud Run, Pub/Sub, Cloud Storage.",
|
||||
"attachments": {},
|
||||
}
|
||||
|
||||
result = await score_execution(
|
||||
CASE, execution(product_selection=bare), HEADINGS, passing_terraform
|
||||
)
|
||||
|
||||
assert not check(result, "selection_has_alternatives").passed
|
||||
assert not check(result, "selection_cites_documentation").passed
|
||||
assert check(result, "required_products_selected").passed
|
||||
|
||||
|
||||
async def test_invalid_terraform_and_diagram_fail_and_warnings_fail_completion():
|
||||
diagram = {"content": "d", "attachments": {"architecture.mmd": "A --> B"}}
|
||||
|
||||
result = await score_execution(
|
||||
CASE,
|
||||
execution("completed_with_warnings", architecture_diagram=diagram),
|
||||
HEADINGS,
|
||||
failing_terraform,
|
||||
)
|
||||
|
||||
assert not check(result, "completed").passed
|
||||
assert not check(result, "terraform_validates").passed
|
||||
assert not check(result, "diagram_is_valid_mermaid").passed
|
||||
assert not result.passed
|
||||
|
||||
|
||||
async def test_an_execution_that_never_produced_deliverables_scores_zero_safely():
|
||||
empty = Interaction("exec-2", "completed_with_warnings", None, {"deliverables": {}})
|
||||
|
||||
result = await score_execution(CASE, empty, HEADINGS, passing_terraform)
|
||||
|
||||
assert result.score == 0.0
|
||||
assert not result.passed
|
||||
|
||||
|
||||
def test_benchmark_cases_load_with_valid_answer_patterns():
|
||||
import re
|
||||
|
||||
from eval.scoring import load_cases
|
||||
|
||||
cases = load_cases(Path(__file__).parents[2] / "eval/datasets/solution_cases.json")
|
||||
|
||||
assert len(cases) >= 2
|
||||
for case in cases:
|
||||
assert case.required_products
|
||||
for pattern, answer in case.scripted_answers:
|
||||
re.compile(pattern)
|
||||
assert answer
|
||||
Reference in New Issue
Block a user