Some checks failed
validation / verify (push) Failing after 13s
- Schema review: a dotted path is judged only when it starts at a name the resource type has, so a database flag or a service name is no longer read as a missing setting. Lesson L-04, which worked around that, is retired. - Skill routing: in a Change request the question says what changed. - Evaluation: a case may list code_standards the infrastructure code must meet; case 06 checks its overlay's regions, naming and labels. - Model tiers (iteration 5 step 6): a lighter model for a role's first attempt, named in the provider bundle. Measured on case 03 and left off: Flash saved five cents on the IaC author and took twice the requests. - Steps 7 and 9 decided against on the numbers, recorded in the plans. - Plans: the runs of cases 05 and 06, and the first proof of the lessons on case 03 (no Revision, 22 minutes against 28, quality held). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
579 lines
21 KiB
Python
579 lines
21 KiB
Python
"""Score a finished Execution against a benchmark case.
|
|
|
|
Each check is a small, explainable rule tied to the upstream Skill's requirements.
|
|
A case passes when its score reaches PASS_THRESHOLD.
|
|
|
|
Structural checks always run. Judged checks (pillar checklist coverage, recommendation
|
|
quality) run when a judge is given, and are what lets a better or worse Skill move the
|
|
score. Thresholds on revisions, seconds and tokens turn a slower or costlier run into a
|
|
reported regression without changing the score.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from app.adk.runtime import ToolResult
|
|
from app.agent import Interaction
|
|
from app.tools.assessment import QUALITY_BAR, QUALITY_RUBRIC
|
|
from app.tools.local import mermaid_check
|
|
from app.workflows.chain import (
|
|
COMPONENTS_FILE,
|
|
PRODUCTS_FILE,
|
|
RECOMMENDATIONS_FILE,
|
|
REQUIREMENTS_FILE,
|
|
VERDICTS_FILE,
|
|
read_graph,
|
|
read_list,
|
|
validation_outcome,
|
|
)
|
|
from app.workflows.deliverables import DeliverableKind
|
|
from app.workflows.target_architecture import (
|
|
DIAGRAM_ATTACHMENT,
|
|
GRAPH_ATTACHMENT,
|
|
GraphError,
|
|
chain_gaps,
|
|
parse_graph,
|
|
render_mermaid,
|
|
)
|
|
from eval.judge import Judge, JudgeUnavailable
|
|
|
|
PASS_THRESHOLD = 0.8
|
|
CHECKLIST_COVERAGE = 0.7 # share of a pillar Skill's checklist the design must address
|
|
THRESHOLD_METRICS = ("revisions", "seconds", "tokens")
|
|
|
|
TerraformValidate = Callable[[Mapping[str, str]], Awaitable[ToolResult]]
|
|
|
|
_HEADING = re.compile(r"^(#{2,6})\s+(.+?)\s*$", re.MULTILINE)
|
|
_NUMBERING = re.compile(r"^\d+(?:\.\d+)*\.?\s+")
|
|
_DOC_LINK = re.compile(r"https://(?:docs\.)?cloud\.google\.com/\S+")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CaseSpec:
|
|
id: str
|
|
name: str
|
|
workflow_request: str
|
|
required_products: tuple[str, ...]
|
|
scripted_answers: tuple[tuple[str, ...], ...] # (pattern, answer[, asker])
|
|
# Run with a user in the loop (scripted): Approvals and questions are answered
|
|
# turn by turn, and Specialists are checked for asking before they submit.
|
|
interactive: bool = False
|
|
# Upper bounds for "revisions", "seconds" and "tokens"; exceeding one is a regression.
|
|
thresholds: Mapping[str, float] = field(default_factory=dict)
|
|
# A change case: the request is run to completion, then this statement of what
|
|
# has changed is run as a Change request against it (ADR-0006).
|
|
change: str = ""
|
|
# Overlay Skills the case's Executions name (ADR-0004).
|
|
overlay_skills: tuple[str, ...] = ()
|
|
# What the infrastructure code must and must not contain, as regular
|
|
# expressions over all of it: how a case checks that standards from an overlay
|
|
# Skill reached the code. Each is {"name", "require"} or {"name", "forbid"}.
|
|
code_standards: tuple[Mapping[str, str], ...] = ()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Check:
|
|
name: str
|
|
passed: bool
|
|
detail: str = ""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CaseResult:
|
|
case_id: str
|
|
status: str
|
|
checks: tuple[Check, ...]
|
|
details: Mapping[str, Any] = field(default_factory=dict)
|
|
|
|
@property
|
|
def score(self) -> float:
|
|
return (
|
|
sum(c.passed for c in self.checks) / len(self.checks)
|
|
if self.checks
|
|
else 0.0
|
|
)
|
|
|
|
@property
|
|
def passed(self) -> bool:
|
|
return self.score >= PASS_THRESHOLD
|
|
|
|
|
|
def load_cases(path: Path) -> list[CaseSpec]:
|
|
return [
|
|
CaseSpec(
|
|
id=item["id"],
|
|
name=item["name"],
|
|
workflow_request=item["workflow_request"],
|
|
required_products=tuple(item["required_products"]),
|
|
scripted_answers=tuple(
|
|
tuple(pair) for pair in item.get("scripted_answers", [])
|
|
),
|
|
interactive=bool(item.get("interactive", False)),
|
|
thresholds=dict(item.get("thresholds", {})),
|
|
change=str(item.get("change", "")),
|
|
overlay_skills=tuple(item.get("overlay_skills", ())),
|
|
code_standards=tuple(item.get("code_standards", ())),
|
|
)
|
|
for item in json.loads(path.read_text(encoding="utf-8"))
|
|
]
|
|
|
|
|
|
def template_headings(template: Path) -> list[str]:
|
|
"""Section headings (level 2+) of a solution-guide template, placeholders removed."""
|
|
return [
|
|
heading
|
|
for _, heading in _HEADING.findall(template.read_text(encoding="utf-8"))
|
|
if "[" not in heading
|
|
]
|
|
|
|
|
|
def checklist_items(skill_file: Path) -> list[str]:
|
|
"""The bullet items under a pillar Skill's "Validation checklist" heading."""
|
|
text = skill_file.read_text(encoding="utf-8")
|
|
match = re.search(
|
|
r"^##\s+Validation checklist\s*$(.*?)(?=^##\s|\Z)",
|
|
text,
|
|
re.MULTILINE | re.DOTALL,
|
|
)
|
|
if not match:
|
|
return []
|
|
# Bullets may nest: a bullet with deeper bullets under it is a category, and the
|
|
# items to check are the leaves, each prefixed with its category.
|
|
bullets: list[tuple[int, str]] = []
|
|
for line in match.group(1).splitlines():
|
|
bullet = re.match(r"^(\s*)[-*]\s+(.*)$", line)
|
|
if bullet:
|
|
text = re.sub(r"^\[[ xX]?\]\s*", "", bullet.group(2).strip())
|
|
bullets.append((len(bullet.group(1)), text.replace("**", "")))
|
|
elif bullets and line.startswith(" ") and line.strip():
|
|
indent, text = bullets[-1]
|
|
bullets[-1] = (indent, f"{text} {line.strip()}")
|
|
|
|
items: list[str] = []
|
|
category = ""
|
|
for index, (indent, text) in enumerate(bullets):
|
|
has_children = index + 1 < len(bullets) and bullets[index + 1][0] > indent
|
|
if has_children:
|
|
category = text.rstrip(":").strip()
|
|
elif indent == 0:
|
|
category = ""
|
|
items.append(text)
|
|
else:
|
|
items.append(f"{category}: {text}" if category else text)
|
|
return items
|
|
|
|
|
|
def regressions(case: CaseSpec, measured: Mapping[str, float | None]) -> list[str]:
|
|
"""Each threshold the run exceeded, as "metric: measured > limit"."""
|
|
over = []
|
|
for metric in THRESHOLD_METRICS:
|
|
limit = case.thresholds.get(metric)
|
|
value = measured.get(metric)
|
|
if limit is not None and value is not None and value > limit:
|
|
over.append(f"{metric}: {value:g} > {limit:g}")
|
|
return over
|
|
|
|
|
|
def _normalise(text: str) -> str:
|
|
return " ".join(_NUMBERING.sub("", text.strip().lower()).split())
|
|
|
|
|
|
def _mentions(text: str, product: str) -> bool:
|
|
return (
|
|
re.search(rf"(?<![\w/]){re.escape(product.lower())}(?![\w/])", text.lower())
|
|
is not None
|
|
)
|
|
|
|
|
|
async def score_execution(
|
|
case: CaseSpec,
|
|
interaction: Interaction,
|
|
headings: Sequence[str],
|
|
terraform_validate: TerraformValidate,
|
|
*,
|
|
judge: Judge | None = None,
|
|
pillar_checklists: Mapping[str, Sequence[str]] | None = None,
|
|
timeline: Sequence[Mapping[str, Any]] = (),
|
|
) -> CaseResult:
|
|
deliverables: Mapping[str, Mapping[str, Any]] = interaction.state.get(
|
|
"deliverables", {}
|
|
)
|
|
|
|
def content(kind: DeliverableKind) -> str:
|
|
item = deliverables.get(kind.value)
|
|
return item["content"] if item else ""
|
|
|
|
def attachments(kind: DeliverableKind) -> Mapping[str, str]:
|
|
item = deliverables.get(kind.value)
|
|
return item.get("attachments", {}) if item else {}
|
|
|
|
checks: list[Check] = []
|
|
|
|
checks.append(
|
|
Check(
|
|
"completed",
|
|
interaction.status == "completed",
|
|
"; ".join(interaction.state.get("warnings", [])) or interaction.status,
|
|
)
|
|
)
|
|
|
|
guide = content(DeliverableKind.SOLUTION_GUIDE)
|
|
guide_headings = {_normalise(h) for _, h in _HEADING.findall(guide)}
|
|
missing_sections = [h for h in headings if _normalise(h) not in guide_headings]
|
|
checks.append(
|
|
Check(
|
|
"guide_follows_template",
|
|
bool(guide) and not missing_sections,
|
|
"missing: " + "; ".join(missing_sections) if missing_sections else "",
|
|
)
|
|
)
|
|
|
|
selection = content(DeliverableKind.PRODUCT_SELECTION)
|
|
selected_text = selection + "\n" + guide
|
|
missing_products = [
|
|
p for p in case.required_products if not _mentions(selected_text, p)
|
|
]
|
|
checks.append(
|
|
Check(
|
|
"required_products_selected",
|
|
bool(selection) and not missing_products,
|
|
"missing: " + ", ".join(missing_products) if missing_products else "",
|
|
)
|
|
)
|
|
|
|
lowered = selection.lower()
|
|
checks.append(
|
|
Check(
|
|
"selection_has_alternatives",
|
|
"alternative" in lowered and "pros" in lowered and "cons" in lowered,
|
|
)
|
|
)
|
|
checks.append(
|
|
Check(
|
|
"selection_cites_documentation",
|
|
bool(_DOC_LINK.search(selection)),
|
|
)
|
|
)
|
|
|
|
requirements = content(DeliverableKind.REQUIREMENTS)
|
|
leaked = [
|
|
product
|
|
for product in case.required_products
|
|
if _mentions(requirements, product)
|
|
and not _mentions(case.workflow_request, product)
|
|
]
|
|
checks.append(
|
|
Check(
|
|
"requirements_name_no_products",
|
|
bool(requirements) and not leaked,
|
|
"names: " + ", ".join(leaked) if leaked else "",
|
|
)
|
|
)
|
|
|
|
diagram = attachments(DeliverableKind.TARGET_ARCHITECTURE).get(
|
|
"architecture.mmd", ""
|
|
)
|
|
mermaid = await mermaid_check(diagram)
|
|
checks.append(
|
|
Check(
|
|
"diagram_is_valid_mermaid",
|
|
bool(diagram) and mermaid.message.startswith("Mermaid check passed"),
|
|
mermaid.message,
|
|
)
|
|
)
|
|
|
|
terraform_files = {
|
|
name.removeprefix("terraform/"): text
|
|
for name, text in attachments(DeliverableKind.DEPLOYMENT_GUIDANCE).items()
|
|
if name.endswith((".tf", ".tfvars", ".tf.json"))
|
|
}
|
|
if terraform_files:
|
|
validation = await terraform_validate(terraform_files)
|
|
terraform_ok = validation.message.startswith("terraform validate passed")
|
|
terraform_detail = validation.message[:1500]
|
|
else:
|
|
terraform_ok, terraform_detail = False, "no Terraform files attached"
|
|
checks.append(Check("terraform_validates", terraform_ok, terraform_detail))
|
|
if case.code_standards:
|
|
checks.append(_code_standards(case.code_standards, terraform_files))
|
|
|
|
checks.append(_chain_check(deliverables))
|
|
checks.append(_verdicts_check(attachments(DeliverableKind.VALIDATION_RESULTS)))
|
|
checks.append(
|
|
_diagram_matches_graph(attachments(DeliverableKind.TARGET_ARCHITECTURE))
|
|
)
|
|
|
|
if case.interactive:
|
|
checks.append(_asks_before_submitting(timeline))
|
|
|
|
if judge is not None:
|
|
recommendations = content(DeliverableKind.DESIGN_RECOMMENDATIONS)
|
|
checks.append(
|
|
await _checklist_coverage(judge, recommendations, pillar_checklists or {})
|
|
)
|
|
checks.append(await _quality(judge, recommendations))
|
|
|
|
return CaseResult(case.id, interaction.status, tuple(checks))
|
|
|
|
|
|
def _code_standards(
|
|
standards: Sequence[Mapping[str, str]], files: Mapping[str, str]
|
|
) -> Check:
|
|
"""Whether the code holds what the case requires of it and nothing it forbids."""
|
|
code = "\n".join(files.values())
|
|
broken = []
|
|
for standard in standards:
|
|
required, forbidden = standard.get("require"), standard.get("forbid")
|
|
if required and not re.search(required, code):
|
|
broken.append(f"{standard['name']}: not found")
|
|
found = re.search(forbidden, code) if forbidden else None
|
|
if found:
|
|
broken.append(f"{standard['name']}: found {found.group(0)}")
|
|
return Check("code_follows_standards", bool(code) and not broken, "; ".join(broken))
|
|
|
|
|
|
def _chain_check(deliverables: Mapping[str, Mapping[str, Any]]) -> Check:
|
|
"""The Requirements chain holds end to end: every requirement reaches a Check,
|
|
directly or through a component's Element, and nothing along it is missing."""
|
|
name = "requirements_chain_complete"
|
|
graph = read_graph(deliverables)
|
|
if graph is None:
|
|
return Check(name, False, "no well-formed Target architecture")
|
|
gaps = chain_gaps(
|
|
graph,
|
|
requirements=read_list(deliverables, "requirements", REQUIREMENTS_FILE),
|
|
components=read_list(deliverables, "technical_decomposition", COMPONENTS_FILE),
|
|
products=read_list(deliverables, "product_selection", PRODUCTS_FILE),
|
|
recommendations=read_list(
|
|
deliverables, "design_recommendations", RECOMMENDATIONS_FILE
|
|
),
|
|
)
|
|
return Check(name, not gaps, "; ".join(gaps))
|
|
|
|
|
|
def _verdicts_check(attachments: Mapping[str, str]) -> Check:
|
|
"""Validation ended with every Check passed and the dry-run not failed."""
|
|
name = "every_check_passes"
|
|
text = attachments.get(VERDICTS_FILE)
|
|
if text is None:
|
|
return Check(name, False, f"no {VERDICTS_FILE}")
|
|
outcome = validation_outcome(text)
|
|
open_checks = [f"{item['check']} failed" for item in outcome["failed"]] + [
|
|
f"{item['check']} not verifiable" for item in outcome["not_verifiable"]
|
|
]
|
|
if outcome["dry_run"].get("verdict") == "fail":
|
|
open_checks.append("dry-run failed")
|
|
if not outcome["passed"] and not open_checks:
|
|
open_checks.append("no verdicts")
|
|
return Check(name, not open_checks, "; ".join(open_checks))
|
|
|
|
|
|
def _diagram_matches_graph(attachments: Mapping[str, str]) -> Check:
|
|
name = "diagram_matches_graph"
|
|
try:
|
|
graph = parse_graph(attachments.get(GRAPH_ATTACHMENT, ""))
|
|
except GraphError as exc:
|
|
return Check(name, False, exc.problems[0])
|
|
matches = attachments.get(DIAGRAM_ATTACHMENT) == render_mermaid(graph)
|
|
return Check(name, matches, "" if matches else "the diagram is not the graph's")
|
|
|
|
|
|
def _asks_before_submitting(timeline: Sequence[Mapping[str, Any]]) -> Check:
|
|
"""In an interactive Execution every Specialist confirms with the user before its
|
|
first submission (spec risk D32)."""
|
|
asked: set[str] = set()
|
|
silent: list[str] = []
|
|
seen: set[str] = set()
|
|
for run in timeline:
|
|
deliverable = run.get("deliverable")
|
|
if not deliverable:
|
|
continue
|
|
if run.get("outcome") == "NeedsInput":
|
|
asked.add(deliverable)
|
|
elif run.get("outcome") == "Submitted" and deliverable not in seen:
|
|
seen.add(deliverable)
|
|
if deliverable not in asked:
|
|
silent.append(deliverable)
|
|
return Check(
|
|
"specialists_ask_before_submitting",
|
|
bool(seen) and not silent,
|
|
"submitted without asking: " + ", ".join(silent) if silent else "",
|
|
)
|
|
|
|
|
|
async def _checklist_coverage(
|
|
judge: Judge, recommendations: str, checklists: Mapping[str, Sequence[str]]
|
|
) -> Check:
|
|
name = "pillar_checklists_covered"
|
|
questions = {
|
|
f"{pillar}.{index}": {
|
|
"kind": "yes_no",
|
|
"text": f"Do the recommendations address this {pillar} checklist item, "
|
|
f"either by recommending it or by explaining why it does not apply? {item}",
|
|
}
|
|
for pillar, items in checklists.items()
|
|
for index, item in enumerate(items)
|
|
}
|
|
if not recommendations or not questions:
|
|
return Check(name, False, "no design recommendations or no checklists")
|
|
try:
|
|
scores = await judge.answer(recommendations, questions)
|
|
except JudgeUnavailable as exc:
|
|
return Check(name, False, f"judge unavailable: {exc}")
|
|
per_pillar = {
|
|
pillar: sum(scores.get(f"{pillar}.{i}", 0) for i in range(len(items)))
|
|
/ len(items)
|
|
for pillar, items in checklists.items()
|
|
if items
|
|
}
|
|
weakest = min(per_pillar.values())
|
|
detail = ", ".join(f"{pillar} {share:.0%}" for pillar, share in per_pillar.items())
|
|
return Check(name, weakest >= CHECKLIST_COVERAGE, detail)
|
|
|
|
|
|
async def _quality(judge: Judge, recommendations: str) -> Check:
|
|
name = "recommendations_quality"
|
|
if not recommendations:
|
|
return Check(name, False, "no design recommendations")
|
|
try:
|
|
scores = await judge.answer(
|
|
recommendations,
|
|
{
|
|
key: {"kind": "graded", "text": text}
|
|
for key, text in QUALITY_RUBRIC.items()
|
|
},
|
|
)
|
|
except JudgeUnavailable as exc:
|
|
return Check(name, False, f"judge unavailable: {exc}")
|
|
mean = sum(scores.values()) / len(QUALITY_RUBRIC)
|
|
detail = ", ".join(f"{key} {scores.get(key, 0):.2f}" for key in QUALITY_RUBRIC)
|
|
return Check(name, mean >= QUALITY_BAR, f"mean {mean:.2f}: {detail}")
|
|
|
|
|
|
# --- a Change request stays contained (iteration 7, step 7) ------------------------------
|
|
|
|
UNREQUESTED_SHARE = 0.1 # of the Baseline graph's items, changed without being reached
|
|
|
|
|
|
def score_change(
|
|
interaction: Interaction, baseline_tokens: float | None, tokens: float | None
|
|
) -> list[Check]:
|
|
"""Checks on a Change request against its Baseline.
|
|
|
|
They read what the Execution recorded: the requirements' own account of what
|
|
changed, and a comparison by id of the two versions' graphs.
|
|
"""
|
|
from app.workflows.changes import ADDED, CHANGED, REMOVED, change_summary
|
|
|
|
state = interaction.state
|
|
baseline = (state.get("baseline") or {}).get("deliverables") or {}
|
|
deliverables = state.get("deliverables", {})
|
|
summary = change_summary(baseline, deliverables)
|
|
requirements = summary["requirements"]
|
|
checks = [
|
|
Check(
|
|
"change_is_recorded",
|
|
bool(baseline) and bool(requirements),
|
|
", ".join(f"{item['id']} {item['change']}" for item in requirements)
|
|
or "no requirement is marked added, changed or removed",
|
|
)
|
|
]
|
|
|
|
graph = _graph_data(deliverables)
|
|
touched = set(summary["checks"][ADDED]) | set(summary["checks"][CHANGED])
|
|
removed_checks = set(summary["checks"][REMOVED])
|
|
old_sources = {
|
|
item.get("id"): item.get("source") for item in _graph_data(baseline)["checks"]
|
|
}
|
|
serving: dict[str, set[str]] = {}
|
|
for component in read_list(
|
|
deliverables, "technical_decomposition", COMPONENTS_FILE
|
|
):
|
|
for requirement in component.get("requirements") or ():
|
|
serving.setdefault(str(requirement), set()).add(str(component.get("id")))
|
|
unreached = []
|
|
for item in requirements:
|
|
identifier = item["id"]
|
|
if item["change"] == REMOVED:
|
|
# What it asked for should go: a Check that came from it is removed.
|
|
reaches = any(old_sources.get(c) == identifier for c in removed_checks)
|
|
reaches = reaches or identifier not in old_sources.values()
|
|
else:
|
|
# A constraint is the source of its Checks. A functional requirement is
|
|
# carried by the components that serve it, so it is reached through a
|
|
# Check on an Element that realises one of them.
|
|
realising = {
|
|
element.get("id")
|
|
for element in graph["elements"]
|
|
if element.get("component") in serving.get(identifier, ())
|
|
}
|
|
reaches = any(
|
|
check.get("id") in touched
|
|
and (
|
|
check.get("source") == identifier
|
|
or check.get("applies_to") in realising
|
|
)
|
|
for check in graph["checks"]
|
|
)
|
|
if not reaches:
|
|
unreached.append(identifier)
|
|
checks.append(
|
|
Check(
|
|
"changed_requirements_reach_checks",
|
|
bool(requirements) and not unreached,
|
|
"no Check changed for: " + ", ".join(unreached) if unreached else "",
|
|
)
|
|
)
|
|
|
|
unrequested = [
|
|
f"{part} {identifier}"
|
|
for part, ids in summary["unrequested"].items()
|
|
for identifier in ids
|
|
]
|
|
size = sum(len(items) for items in _graph_data(baseline).values()) or 1
|
|
checks.append(
|
|
Check(
|
|
"change_is_contained",
|
|
len(unrequested) <= max(2, UNREQUESTED_SHARE * size),
|
|
f"{len(unrequested)} of {size} items changed without being reached"
|
|
+ (": " + ", ".join(unrequested[:12]) if unrequested else ""),
|
|
)
|
|
)
|
|
cheaper = (
|
|
baseline_tokens is not None and tokens is not None and tokens < baseline_tokens
|
|
)
|
|
checks.append(
|
|
Check(
|
|
"change_costs_less_than_baseline",
|
|
cheaper,
|
|
f"{tokens or 0:,.0f} tokens against {baseline_tokens or 0:,.0f}",
|
|
)
|
|
)
|
|
return checks
|
|
|
|
|
|
def _graph_data(
|
|
deliverables: Mapping[str, Mapping[str, Any]],
|
|
) -> dict[str, list[dict[str, Any]]]:
|
|
text = (
|
|
(deliverables.get(DeliverableKind.TARGET_ARCHITECTURE.value) or {})
|
|
.get("attachments", {})
|
|
.get("target-architecture.json", "")
|
|
)
|
|
try:
|
|
data = json.loads(text)
|
|
except ValueError:
|
|
data = {}
|
|
if not isinstance(data, dict):
|
|
data = {}
|
|
return {
|
|
part: [item for item in data.get(part, []) if isinstance(item, dict)]
|
|
for part in ("elements", "relationships", "checks")
|
|
}
|