latest refactoring
Some checks failed
validation / verify (push) Failing after 10s

This commit is contained in:
2026-10-06 20:22:37 +01:00
parent e94b2493e7
commit 7ee49ab352
414 changed files with 15768 additions and 3896 deletions

View File

@@ -8,15 +8,14 @@ import pytest
from langgraph.checkpoint.memory import InMemorySaver
from starlette.testclient import TestClient
from app.a2a.server import create_app
from app.agents.fake_runtime import Call, FakeAgentRuntime
from app.agents.roles import load_roles
from app.assembly import Assembly
from app.adk.fake_runtime import Call, FakeAgentRuntime
from app.adk.roles import load_roles
from app.agent import Assembly, ExecutionEngine
from app.config import get_settings
from app.execution.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
from app.execution.engine import ExecutionEngine
from app.execution.recorder import InMemoryRecorder
from app.main import create_app
from app.providers import load_provider
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
AGENTS_DIR = Path(__file__).parents[2] / "agents"
K = DeliverableKind

View File

@@ -11,13 +11,13 @@ from google.genai import types
from langgraph.checkpoint.memory import InMemorySaver
from pydantic import Field
from app.agents.adk_runtime import AdkAgentRuntime
from app.agents.roles import load_roles
from app.execution.deliverables import DEPENDENCY_GRAPH, DeliverableKind
from app.execution.engine import ExecutionEngine
from app.execution.recorder import InMemoryRecorder
from app.execution.toolkit import make_tool_factory
from app.adk.adk_runtime import AdkAgentRuntime
from app.adk.roles import load_roles
from app.agent import ExecutionEngine
from app.providers import load_provider
from app.recorder import InMemoryRecorder
from app.tools.toolkit import make_tool_factory
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
AGENTS_DIR = Path(__file__).parents[2] / "agents"
@@ -47,7 +47,7 @@ def requirements_markdown() -> str:
def task_context(request: LlmRequest) -> dict:
text = request.contents[0].parts[0].text
text = (request.contents[0].parts or [])[0].text or ""
return json.loads(text.split("```json\n", 1)[1].rsplit("\n```", 1)[0])

View File

@@ -3,16 +3,18 @@
from collections.abc import AsyncGenerator
from pathlib import Path
import httpx
import pytest
from google.adk.models.base_llm import BaseLlm
from google.adk.models.google_llm import Gemini
from google.adk.models.lite_llm import LiteLlm
from google.adk.models.llm_request import LlmRequest
from google.adk.models.llm_response import LlmResponse
from google.genai import types
from pydantic import Field
from app.agents.adk_runtime import AdkAgentRuntime, default_model_factory
from app.agents.runtime import (
from app.adk.adk_runtime import AdkAgentRuntime, default_model_factory
from app.adk.runtime import (
AgentTask,
BudgetExhausted,
NeedsInput,
@@ -197,12 +199,120 @@ async def test_budget_counts_every_tool_call_and_stops_the_run():
assert len(recorder.calls) == 2
async def test_model_reply_without_a_turn_ending_tool_is_stopped():
model = ScriptedLlm(script=[[text("I need the RTO before I can continue.")]])
async def test_model_reply_without_a_turn_ending_tool_is_stopped_after_nudges():
model = ScriptedLlm(
script=[[text("I need the RTO before I can continue.")], [text("")], [text("")]]
)
outcome = await runtime_with(model).run(ROLE, TASK, [])
assert outcome == Stopped("I need the RTO before I can continue.")
assert len(model.requests) == 3
assert "without calling a tool that ends your turn" in (
model.requests[1].contents[-1].parts[0].text
)
async def test_a_nudged_agent_that_then_ends_its_turn_succeeds_in_one_session():
recorder = Recorder()
model = ScriptedLlm(script=[[text("")], [call("delegate", note="go")]])
outcome = await runtime_with(model).run(
RoleSpec("orchestrator", "Coordinate.", budget=5),
TASK,
[recorder.tool("delegate", TurnEnded("delegate"))],
)
assert outcome == TurnEnded("delegate")
assert "without calling a tool" in model.requests[1].contents[-1].parts[0].text
class TimingOutLlm(ScriptedLlm):
"""Times out on the first request, then follows its script."""
timed_out: bool = False
async def generate_content_async(
self, llm_request: LlmRequest, stream: bool = False
) -> AsyncGenerator[LlmResponse, None]:
if not self.timed_out:
self.timed_out = True
raise httpx.ReadTimeout("no response")
async for response in super().generate_content_async(llm_request, stream):
yield response
async def test_a_timed_out_model_request_is_retried_in_the_same_session():
recorder = Recorder()
model = TimingOutLlm(script=[[call("delegate", note="go")]])
outcome = await runtime_with(model).run(
RoleSpec("orchestrator", "Coordinate.", budget=5),
TASK,
[recorder.tool("delegate", TurnEnded("delegate"))],
)
assert outcome == TurnEnded("delegate")
assert "timed out" in model.requests[0].contents[-1].parts[0].text
async def test_model_timeout_is_set_on_every_request():
model = ScriptedLlm(script=[[call("delegate")]])
runtime = AdkAgentRuntime(
skill_dirs=SKILLS, model_factory=lambda role: model, model_timeout_seconds=90
)
await runtime.run(
RoleSpec("orchestrator", "Coordinate.", budget=5),
TASK,
[Recorder().tool("delegate", TurnEnded("delegate"))],
)
assert model.requests[0].config.http_options.timeout == 90_000
class MeteredLlm(ScriptedLlm):
async def generate_content_async(
self, llm_request: LlmRequest, stream: bool = False
) -> AsyncGenerator[LlmResponse, None]:
async for response in super().generate_content_async(llm_request, stream):
response.usage_metadata = types.GenerateContentResponseUsageMetadata(
prompt_token_count=100,
cached_content_token_count=40,
candidates_token_count=10,
thoughts_token_count=5,
total_token_count=115,
)
yield response
async def test_token_usage_is_summed_per_role_and_model():
model = MeteredLlm(script=[[text("")], [call("delegate")]])
role = RoleSpec("orchestrator", "Coordinate.", budget=5, model="gemini/x")
runtime = runtime_with(model)
await runtime.run(role, TASK, [Recorder().tool("delegate", TurnEnded("delegate"))])
usage = runtime.usage[("orchestrator", "gemini/x")]
assert (usage.requests, usage.prompt, usage.cached, usage.output) == (
2,
200,
80,
20,
)
assert (usage.thinking, usage.total) == (10, 230)
async def test_task_message_names_the_roles_skills():
model = ScriptedLlm(script=[[text("done")]] * 3)
await runtime_with(model).run(ROLE, TASK, [])
first = model.requests[0].contents[0].parts[0].text
assert (
"Skills available to you (load with load_skill): google-cloud-solution-architecture."
in first
)
async def test_needs_input_outcome_passes_through():
@@ -218,7 +328,9 @@ async def test_needs_input_outcome_passes_through():
async def test_each_run_starts_with_a_fresh_session():
model = ScriptedLlm(script=[[text("first")], [text("second")]])
runtime = runtime_with(model)
runtime = AdkAgentRuntime(
skill_dirs=SKILLS, model_factory=lambda role: model, max_nudges=0
)
await runtime.run(ROLE, TASK, [])
await runtime.run(ROLE, TASK, [])
@@ -226,13 +338,19 @@ async def test_each_run_starts_with_a_fresh_session():
assert len(model.requests[1].contents) == 1
def test_default_model_factory_wraps_the_role_model_in_litellm():
model = default_model_factory(
RoleSpec("diagrammer", "", budget=1, model="gemini/gemini-flash-latest")
def test_default_model_factory_uses_native_gemini_and_litellm_for_others():
gemini = default_model_factory(
RoleSpec("diagrammer", "", budget=1, model="gemini/gemini-3.8-flash")
)
other = default_model_factory(
RoleSpec("diagrammer", "", budget=1, model="anthropic/claude-sonnet-5")
)
assert isinstance(model, LiteLlm)
assert model.model == "gemini/gemini-flash-latest"
assert isinstance(gemini, Gemini)
assert gemini.model == "gemini-3.8-flash"
assert gemini.retry_options.attempts == 5
assert isinstance(other, LiteLlm)
assert other.model == "anthropic/claude-sonnet-5"
with pytest.raises(ValueError, match="model"):
default_model_factory(RoleSpec("diagrammer", "", budget=1))
@@ -245,7 +363,7 @@ def test_unknown_skill_is_rejected_when_the_runtime_is_built():
)
def _function_responses(request: LlmRequest) -> dict[str, dict]:
def _function_responses(request: LlmRequest) -> dict:
responses = {}
for content in request.contents:
for part in content.parts or []:

View File

@@ -4,9 +4,9 @@ from pathlib import Path
import pytest
from app.agents.roles import PILLAR_ADVISOR, RoleConfigError, load_roles
from app.execution.deliverables import DEPENDENCY_GRAPH
from app.adk.roles import PILLAR_ADVISOR, RoleConfigError, load_roles
from app.providers import load_provider
from app.workflows.deliverables import DEPENDENCY_GRAPH
AGENTS_DIR = Path(__file__).parents[2] / "agents"
@@ -34,7 +34,8 @@ def test_roles_take_skills_mcp_servers_and_models_from_the_bundle(bundle):
assert roles["product-selector"].mcp_servers == ("developer_knowledge",)
assert "asset_inventory" in roles["requirements-analyst"].mcp_servers
assert roles["diagrammer"].model == bundle.models.for_role("diagrammer")
assert roles["orchestrator"].model == bundle.models.default
assert roles["orchestrator"].model == bundle.models.for_role("orchestrator")
assert roles["decomposer"].model == bundle.models.default
def test_roles_name_only_the_local_tools_they_need(bundle):

0
tests/eval/__init__.py Normal file
View File

183
tests/eval/test_scoring.py Normal file
View File

@@ -0,0 +1,183 @@
"""Scoring a finished Execution against a benchmark case, without models or network."""
from pathlib import Path
from app.adk.runtime import ToolResult
from app.agent import Interaction
from eval.scoring import CaseSpec, score_execution, template_headings
TEMPLATE = (
Path(__file__).parents[2]
/ "skills/vendor/google/google-cloud-solution-architecture/assets/output-template.md"
)
HEADINGS = template_headings(TEMPLATE)
CASE = CaseSpec(
id="case-x",
name="Webhook ingestion",
workflow_request="Ingest signed webhooks durably and keep 30 days of raw events.",
required_products=("Cloud Run", "Pub/Sub", "Cloud Storage"),
scripted_answers=(),
)
def guide(headings=HEADINGS) -> str:
return "\n".join(f"## {heading}\nContent." for heading in headings)
def execution(status="completed", **overrides) -> Interaction:
deliverables = {
"requirements": {
"content": "## Functional requirements\nAccept webhooks.\n## Current state\nNone.",
"attachments": {},
},
"product_selection": {
"content": (
"## Products and features mapping\n"
"| Component | Product | Justification | Alternatives | Pros and cons |\n"
"| Ingress | Cloud Run | [docs](https://cloud.google.com/run/docs) | GKE | "
"**Pros**: control **Cons**: toil |\n"
"| Queue | Pub/Sub | ... | Cloud Tasks | ... |\n| Archive | Cloud Storage | ... |"
),
"attachments": {},
},
"architecture_diagram": {
"content": "diagram",
"attachments": {
"architecture.mmd": "flowchart TD\n A[Client] --> B[Run]\n"
},
},
"deployment_guidance": {
"content": "guide",
"attachments": {
"terraform/main.tf": "resource {}",
"terraform/variables.tf": "",
},
},
"solution_guide": {"content": guide(), "attachments": {}},
}
for kind, value in overrides.items():
deliverables[kind] = value
return Interaction(
"exec-1", status, None, {"deliverables": deliverables, "warnings": []}
)
async def passing_terraform(files) -> ToolResult:
return ToolResult("terraform validate passed:\nSuccess!")
async def failing_terraform(files) -> ToolResult:
return ToolResult("terraform validate failed:\nError: Unsupported argument")
def check(result, name):
return next(c for c in result.checks if c.name == name)
def test_template_headings_skip_the_title_and_placeholders():
assert HEADINGS[0] == "1. Executive summary and workload overview"
assert "4.1. Google Cloud products and features mapping" in HEADINGS
assert not any("[" in heading for heading in HEADINGS)
assert len(HEADINGS) >= 20
async def test_a_complete_execution_passes_every_check():
result = await score_execution(CASE, execution(), HEADINGS, passing_terraform)
assert result.passed, [c for c in result.checks if not c.passed]
assert result.score == 1.0
async def test_missing_template_sections_are_named():
partial = {"content": guide(HEADINGS[:-2]), "attachments": {}}
result = await score_execution(
CASE, execution(solution_guide=partial), HEADINGS, passing_terraform
)
detail = check(result, "guide_follows_template").detail
assert not check(result, "guide_follows_template").passed
assert HEADINGS[-1] in detail
async def test_requirements_naming_products_not_in_the_request_breaks_phase_separation():
leaky = {"content": "## Functional requirements\nUse Cloud Run.", "attachments": {}}
result = await score_execution(
CASE, execution(requirements=leaky), HEADINGS, passing_terraform
)
assert not check(result, "requirements_name_no_products").passed
assert "Cloud Run" in check(result, "requirements_name_no_products").detail
async def test_products_named_by_the_user_may_appear_in_requirements():
case = CaseSpec(
**{**CASE.__dict__, "workflow_request": "Use Cloud Run for webhooks."}
)
named = {
"content": "## Functional requirements\nRuns on Cloud Run.",
"attachments": {},
}
result = await score_execution(
case, execution(requirements=named), HEADINGS, passing_terraform
)
assert check(result, "requirements_name_no_products").passed
async def test_selection_without_alternatives_or_citations_fails_those_checks():
bare = {
"content": "## Products and features mapping\nCloud Run, Pub/Sub, Cloud Storage.",
"attachments": {},
}
result = await score_execution(
CASE, execution(product_selection=bare), HEADINGS, passing_terraform
)
assert not check(result, "selection_has_alternatives").passed
assert not check(result, "selection_cites_documentation").passed
assert check(result, "required_products_selected").passed
async def test_invalid_terraform_and_diagram_fail_and_warnings_fail_completion():
diagram = {"content": "d", "attachments": {"architecture.mmd": "A --> B"}}
result = await score_execution(
CASE,
execution("completed_with_warnings", architecture_diagram=diagram),
HEADINGS,
failing_terraform,
)
assert not check(result, "completed").passed
assert not check(result, "terraform_validates").passed
assert not check(result, "diagram_is_valid_mermaid").passed
assert not result.passed
async def test_an_execution_that_never_produced_deliverables_scores_zero_safely():
empty = Interaction("exec-2", "completed_with_warnings", None, {"deliverables": {}})
result = await score_execution(CASE, empty, HEADINGS, passing_terraform)
assert result.score == 0.0
assert not result.passed
def test_benchmark_cases_load_with_valid_answer_patterns():
import re
from eval.scoring import load_cases
cases = load_cases(Path(__file__).parents[2] / "eval/datasets/solution_cases.json")
assert len(cases) >= 2
for case in cases:
assert case.required_products
for pattern, answer in case.scripted_answers:
re.compile(pattern)
assert answer

View File

@@ -2,7 +2,7 @@
import pytest
from app.execution.deliverables import (
from app.workflows.deliverables import (
DEPENDENCY_GRAPH,
PHASE_ORDER,
DeliverableKind,

View File

@@ -9,11 +9,11 @@ import pytest
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from app.agents.fake_runtime import Call, FakeAgentRuntime
from app.agents.runtime import RoleSpec, ToolResult, ToolSpec
from app.execution.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
from app.execution.engine import ExecutionEngine
from app.execution.recorder import InMemoryRecorder
from app.adk.fake_runtime import Call, FakeAgentRuntime
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
from app.agent import ExecutionEngine
from app.recorder import InMemoryRecorder
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
K = DeliverableKind
P = PhaseId
@@ -661,3 +661,28 @@ async def test_a_failing_specialist_is_relayed_to_the_user_as_a_question(tmp_pat
assert question["from"] == "requirements"
assert "model API unavailable" in question["question"]
assert result.state["ledger"]["deliverables"]["requirements"]["status"] == "pending"
async def test_pre_approved_questions_use_matching_scripted_answers(tmp_path):
runtime = FakeAgentRuntime(
{
"orchestrator": [
[ask_user("What is the RTO?", "Which region?")],
[ask_user("Anything else?")],
]
}
)
engine = make_engine(tmp_path, runtime, max_orchestrator_turns=2)
result = await engine.start(
"Ingestion",
pre_approved=True,
scripted_answers=[(r"\bRTO\b", "One hour."), ("region", "europe-west2")],
)
first_reply = result.state["conversation"][2]["text"]
assert "Q: What is the RTO?\nA: One hour." in first_reply
assert "Q: Which region?\nA: europe-west2" in first_reply
assert "No user is available" not in first_reply
second_reply = result.state["conversation"][4]["text"]
assert second_reply.startswith("No user is available")

View File

@@ -0,0 +1,72 @@
"""Files cross tool calls as a list of {path, content}: models rewrite object keys."""
import pytest
from app.adk.runtime import FILES_SCHEMA, Submitted, files_from_argument
from app.tools.execution import specialist_tools
from app.tools.local import TerraformRunner, local_tools
from app.workflows.deliverables import DeliverableKind
GUIDANCE = (
"# Deployment prerequisites\n\nx\n\n# Step-by-step deployment instructions\n\ny\n"
)
def test_files_are_offered_as_a_list_of_path_and_content():
assert FILES_SCHEMA["type"] == "array"
assert FILES_SCHEMA["items"]["required"] == ["path", "content"]
def test_file_arguments_accept_lists_and_maps():
as_list = [{"path": "terraform/main.tf", "content": "a"}]
assert files_from_argument(as_list) == {"terraform/main.tf": "a"}
assert files_from_argument({"terraform/main.tf": "a"}) == {"terraform/main.tf": "a"}
assert files_from_argument(None) == {}
with pytest.raises(TypeError):
files_from_argument(["terraform/main.tf"])
async def test_submission_takes_attachments_as_a_list():
submit = specialist_tools(
DeliverableKind.DEPLOYMENT_GUIDANCE, extra=[], allow_cloud_read=False
)[0]
result = await submit.handler(
content=GUIDANCE,
attachments=[
{"path": "terraform/main.tf", "content": "resource {}"},
{"path": "terraform/variables.tf", "content": ""},
],
)
assert result.outcome == Submitted(
GUIDANCE, {"terraform/main.tf": "resource {}", "terraform/variables.tf": ""}
)
async def test_malformed_attachments_are_refused_to_the_agent():
submit = specialist_tools(
DeliverableKind.DEPLOYMENT_GUIDANCE, extra=[], allow_cloud_read=False
)[0]
result = await submit.handler(content=GUIDANCE, attachments=["terraform/main.tf"])
assert result.outcome is None
assert result.message.startswith("Refused: attachments")
async def test_terraform_tools_take_files_as_a_list():
received = {}
class Runner(TerraformRunner):
async def validate(self, files):
received.update(files)
return await super().validate({})
(validate,) = local_tools(["terraform_validate"], Runner())
await validate.handler(files=[{"path": "terraform/main.tf", "content": "x"}])
assert validate.parameters["properties"]["files"]["type"] == "array"
assert received == {"terraform/main.tf": "x"}

View File

@@ -2,14 +2,14 @@
import pytest
from app.execution.deliverables import DeliverableKind as K
from app.execution.deliverables import PhaseId as P
from app.execution.ledger import (
from app.states.ledger import (
DeliverableStatus,
Ledger,
LedgerRefusal,
PhaseStatus,
)
from app.workflows.deliverables import DeliverableKind as K
from app.workflows.deliverables import PhaseId as P
def produce(ledger: Ledger, *kinds: K) -> None:

View File

@@ -1,10 +1,10 @@
"""The design advisor's fan-out to one advisor per architecture-framework pillar."""
from app.agents.fake_runtime import Call, FakeAgentRuntime
from app.agents.roles import PILLAR_ADVISOR
from app.agents.runtime import RoleSpec, ToolResult, ToolSpec
from app.execution.pillars import CONSULT_TOOL, pillar_consultation_tool
from app.execution.toolkit import make_tool_factory
from app.adk.fake_runtime import Call, FakeAgentRuntime
from app.adk.roles import PILLAR_ADVISOR
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
from app.tools.pillars import CONSULT_TOOL, pillar_consultation_tool
from app.tools.toolkit import make_tool_factory
PILLARS = {
"security": "waf-security",

View File

@@ -2,8 +2,8 @@
import json
from app.execution.engine import Interaction
from app.execution.presentation import describe_prompt, to_generate_response
from app.agent import Interaction
from app.presentation import describe_prompt, to_generate_response
def test_approval_prompt_explains_the_structured_reply():

View File

@@ -6,11 +6,11 @@ import pytest
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from app.checkpointing import open_checkpointer
from app.config import get_settings
from app.database import DatabaseManager
from app.execution.checkpointing import open_checkpointer
from app.execution.ledger import LedgerEvent
from app.execution.recorder import DatabaseRecorder
from app.recorder import DatabaseRecorder
from app.states.ledger import LedgerEvent
@pytest.fixture

View File

@@ -1,212 +0,0 @@
"""Comprehensive test suite for Google ADK Multi-Agent Architecture & PostgreSQL Persistence."""
import pytest
import uuid
from starlette.testclient import TestClient
from app.adk.agents import (
DiscoveryAgent,
ArchitectureDesignAgent,
ValidationReviewAgent,
PackagingAgent,
OrchestratorLoopAgent,
build_adk_multi_agent_system,
)
from app.adk.artifacts import PostgresArtifactRepository
from app.adk.evaluation import ADKEvaluator
from app.adk.runners import ADKAgentRunner
from app.adk.sessions import PostgresSessionService
from app.adk.tools import ADK_TOOLS
from app.adk.workflows import create_gcp_adk_workflow
from app.agent import app
from app.config import get_settings
from app.database import get_db_manager
from app.skills.loader import SkillLoader
@pytest.fixture
def skill_loader():
settings = get_settings()
loader = SkillLoader(settings.SKILLS_DIR)
loader.load_skills()
return loader
@pytest.fixture
def client():
return TestClient(app)
def test_database_manager_operations():
"""Verify PostgreSQL database manager schema initialization & CRUD operations."""
db = get_db_manager()
session_id = f"test-sess-{uuid.uuid4()}"
execution_id = f"test-exec-{uuid.uuid4()}"
# 1. Save and retrieve session
saved = db.save_session(session_id, "TestAgent", {"key": "value"})
assert saved is True
session_data = db.get_session(session_id)
assert session_data is not None
assert session_data["agent_name"] == "TestAgent"
assert session_data["state_data"]["key"] == "value"
# 2. Save workflow execution
saved_exec = db.save_workflow_execution(
execution_id=execution_id,
session_id=session_id,
status="completed",
current_phase="package",
loop_count=1,
request_summary="Unit test workflow request",
results={"passed": True},
)
assert saved_exec is True
# 3. Save artifact
art_saved = db.save_artifact(
artifact_id=f"art-{uuid.uuid4()}",
execution_id=execution_id,
artifact_type="markdown",
file_path="docs/test.md",
content="# Test Content",
)
assert art_saved is True
# 4. Record review log
log_saved = db.record_orchestrator_log(
log_id=f"log-{uuid.uuid4()}",
execution_id=execution_id,
iteration=1,
review_status="APPROVED",
reviewer_agent="ValidationReviewAgent",
feedback="Passed all checks",
)
assert log_saved is True
def test_adk_tools():
"""Verify google.adk.tools wrappers."""
assert len(ADK_TOOLS) == 6
tool_names = [t.name for t in ADK_TOOLS]
assert "validate_mermaid_diagram" in tool_names
assert "validate_terraform_syntax" in tool_names
assert "validate_repository_artifacts" in tool_names
assert "developerknowledge_search_documents" in tool_names
assert "developerknowledge_get_documents" in tool_names
assert "developerknowledge_answer_query" in tool_names
def test_adk_postgres_session_service():
"""Verify PostgresSessionService creating, getting, and saving session state."""
session_service = PostgresSessionService()
session_id = f"adk-session-{uuid.uuid4()}"
sess = session_service.create_session("GCP_ADK_Agent", session_id=session_id)
assert sess.session_id == session_id
assert sess.agent_name == "GCP_ADK_Agent"
sess.state["counter"] = 42
saved = session_service.save_session(sess)
assert saved is True
fetched = session_service.get_session(session_id)
assert fetched is not None
assert fetched.state.get("counter") == 42
def test_adk_orchestrator_loop_agent(skill_loader):
"""Verify OrchestratorLoopAgent multi-agent loop review cycle."""
orchestrator = build_adk_multi_agent_system(skill_loader, max_iterations=3)
assert orchestrator.name == "OrchestratorLoopAgent"
assert orchestrator.max_iterations == 3
initial_state = {
"execution_id": str(uuid.uuid4()),
"workflow_request": "Event-driven regional HTTP application",
"target_dir": ".",
"active_skills": [],
}
final_state = orchestrator.execute(initial_state)
assert final_state.get("validation_passed") is True
assert final_state.get("loop_completed_successfully") is True
assert len(final_state.get("active_skills", [])) == 5
def test_adk_workflow_composition(skill_loader):
"""Verify google.adk.workflows Workflow composition."""
workflow = create_gcp_adk_workflow(skill_loader, max_iterations=2)
assert workflow.name == "GCP_Solution_Architecture_Workflow"
assert len(workflow.steps) == 1
initial_state = {
"execution_id": str(uuid.uuid4()),
"workflow_request": "Event-driven architecture test",
"target_dir": ".",
"active_skills": [],
}
result = workflow.execute(initial_state)
assert result.get("validation_passed") is True
def test_adk_agent_runner():
"""Verify ADKAgentRunner execution and artifact persistence."""
runner = ADKAgentRunner()
session_id = f"runner-session-{uuid.uuid4()}"
res = runner.run_execution(
session_id=session_id,
request_summary="High scale ingestion workflow",
target_dir=".",
)
assert res["status"] == "success"
assert res["validation_passed"] is True
assert "solution_guide" in res["artifacts"]
# Verify session persists in Postgres DB
session_service = PostgresSessionService()
sess = session_service.get_session(session_id)
assert sess is not None
def test_adk_evaluator():
"""Verify ADKEvaluator executing benchmark cases."""
evaluator = ADKEvaluator()
case = {
"id": "case-test",
"workflow_request": "Event-driven regional HTTP application",
"rubric": {
"required_products": ["Cloud Run", "Pub/Sub", "Cloud Storage"],
"requires_mermaid": True,
"requires_terraform": True,
"requires_guide_sections": ["Functional requirements", "Selected products"],
},
}
res = evaluator.evaluate_benchmark_case(case)
assert res["passed"] is True
assert res["percentage"] >= 80.0
def test_starlette_session_api_endpoint(client):
"""Verify Starlette API GET /sessions/{session_id} route."""
session_id = f"api-sess-{uuid.uuid4()}"
# First run workflow to populate session
payload = {
"session_id": session_id,
"request": "Test session retrieval route",
}
gen_resp = client.post("/generate", json=payload)
assert gen_resp.status_code == 200
# Query session
sess_resp = client.get(f"/sessions/{session_id}")
assert sess_resp.status_code == 200
sess_data = sess_resp.json()
assert sess_data["session_id"] == session_id
assert sess_data["agent_name"] == "OrchestratorLoopAgent"

View File

@@ -1,52 +0,0 @@
"""Starlette REST API integration tests using TestClient."""
import pytest
from starlette.testclient import TestClient
from app.agent import app
@pytest.fixture
def client():
return TestClient(app)
def test_health_endpoint(client):
"""GET /health returns healthy status."""
response = client.get("/health")
assert response.status_code == 200
data = response.json()
assert data["status"] == "healthy"
assert "gcp_solution_architecture_agent" in data["agent"]
def test_card_endpoint(client):
"""GET /card returns agent capability card."""
response = client.get("/card")
assert response.status_code == 200
data = response.json()
assert data["name"] == "gcp_solution_architecture_agent"
assert "discover" in data["capabilities"]["phases"]
def test_generate_endpoint(client):
"""POST /generate executes full agent workflow."""
payload = {
"request": "Build an event-driven regional ingestion service",
"target_dir": ".",
}
response = client.post("/generate", json=payload)
assert response.status_code == 200
data = response.json()
assert data["status"] == "success"
assert data["validation_passed"] is True
assert "artifacts" in data
assert "solution_guide" in data["artifacts"]
def test_validate_endpoint(client):
"""POST /validate executes pre-deployment validation checks."""
payload = {"target_dir": "."}
response = client.post("/validate", json=payload)
assert response.status_code == 200
data = response.json()
assert data["valid"] is True

View File

@@ -4,7 +4,7 @@ from pathlib import Path
import pytest
from app.agent_skills import SkillFormatError, discover_skills, read_skill
from app.skills.agent_skills import SkillFormatError, discover_skills, read_skill
def write_skill(

View File

@@ -1,5 +1,4 @@
import pathlib
import re
import unittest
ROOT = pathlib.Path(__file__).parents[1]
@@ -46,7 +45,13 @@ class ArtifactTests(unittest.TestCase):
def test_guide_packages_outputs(self):
guide = get_artifact_text("solution-architecture-guide.md")
for heading in ("Requirements", "Architecture", "Terraform", "Validation", "Verification"):
for heading in (
"Requirements",
"Architecture",
"Terraform",
"Validation",
"Verification",
):
self.assertIn(heading, guide)

View File

@@ -1,13 +1,15 @@
"""Building the engine from settings and the provider bundle."""
from app.agents.fake_runtime import Call, FakeAgentRuntime
from app.assembly import open_assembly
from typing import Any
from app.adk.fake_runtime import Call, FakeAgentRuntime
from app.agent import open_assembly
from app.config import Settings
from app.execution.recorder import InMemoryRecorder
from app.recorder import InMemoryRecorder
def settings(tmp_path, **overrides) -> Settings:
values = {
values: dict[str, Any] = {
"CHECKPOINTER": "memory",
"MCP_ENABLED": False,
"ORCHESTRATOR_BUDGET": 9,

View File

@@ -1,34 +0,0 @@
"""Integration tests for LangGraph StateGraph execution."""
import pytest
from app.workflows.gcp_architecture_graph import create_agent
from app.skills.loader import SkillLoader
from app.config import get_settings
def test_agent_graph_execution():
"""Verify execution of the 4-phase LangGraph StateGraph."""
settings = get_settings()
loader = SkillLoader(settings.SKILLS_DIR)
loader.load_skills()
agent = create_agent(loader)
initial_state = {
"workflow_request": "Event-driven regional HTTP application",
"target_dir": ".",
"active_skills": [],
}
final_state = agent.invoke(initial_state)
assert final_state.get("current_phase") == "package"
assert final_state.get("source_discovery_doc") is not None
assert final_state.get("source_mermaid_diagram") is not None
assert final_state.get("requirements_doc") is not None
assert final_state.get("architecture_doc") is not None
assert final_state.get("mermaid_diagram") is not None
assert final_state.get("terraform_code") is not None
assert final_state.get("validation_results") is not None
assert final_state.get("solution_guide") is not None
assert final_state.get("validation_passed") is True
assert len(final_state.get("active_skills", [])) == 5

View File

@@ -1,53 +0,0 @@
"""Unit & Integration tests for Developer Knowledge MCP Client and Tools."""
import pytest
from app.tools.mcp_developer_knowledge import (
DeveloperKnowledgeMCPClient,
developerknowledge_answer_query,
developerknowledge_get_documents,
developerknowledge_search_documents,
get_mcp_client,
)
from app.adk.tools import mcp_search_tool, mcp_get_tool, mcp_answer_tool
def test_mcp_client_search_documents():
"""Verify DeveloperKnowledgeMCPClient search_documents method."""
client = get_mcp_client()
res = client.search_documents(query="Cloud Run", category="compute")
assert res["source"] in ("live_mcp", "offline_fallback")
assert "documents" in res or "data" in res
def test_mcp_client_get_documents():
"""Verify DeveloperKnowledgeMCPClient get_documents method."""
client = get_mcp_client()
res = client.get_documents(document_uri="https://cloud.google.com/run/docs/overview")
assert res["source"] in ("live_mcp", "offline_fallback")
assert "document" in res or "data" in res
def test_mcp_client_answer_query():
"""Verify DeveloperKnowledgeMCPClient answer_query method."""
client = get_mcp_client()
res = client.answer_query(query="Check Cloud Run release status")
assert res["source"] in ("live_mcp", "offline_fallback")
def test_langchain_mcp_tools():
"""Verify LangChain @tool wrappers for Developer Knowledge MCP."""
search_res = developerknowledge_search_documents.invoke({"query": "PubSub", "category": "messaging"})
assert search_res is not None
get_res = developerknowledge_get_documents.invoke({"document_uri": "https://cloud.google.com/pubsub/docs"})
assert get_res is not None
ans_res = developerknowledge_answer_query.invoke({"query": "What is the release status of Cloud Storage?"})
assert ans_res is not None
def test_adk_mcp_function_tools():
"""Verify ADK FunctionTools for Developer Knowledge MCP."""
assert mcp_search_tool.name == "developerknowledge_search_documents"
assert mcp_get_tool.name == "developerknowledge_get_documents"
assert mcp_answer_tool.name == "developerknowledge_answer_query"

View File

@@ -2,9 +2,21 @@ import subprocess
from pathlib import Path
ROOT = Path(__file__).parents[1]
def test_artifact_validator():
result = subprocess.run(["python3", "scripts/validate_artifacts.py"], cwd=ROOT, capture_output=True, text=True)
result = subprocess.run(
["python3", "scripts/validate_artifacts.py"],
cwd=ROOT,
capture_output=True,
check=False,
text=True,
)
assert result.returncode == 0, result.stdout + result.stderr
def test_no_secret_values():
assert "replace-with-project-id" in (ROOT / "terraform/terraform.tfvars.example").read_text()
assert (
"replace-with-project-id"
in (ROOT / "terraform/terraform.tfvars.example").read_text()
)

View File

@@ -1,50 +0,0 @@
"""Unit tests for Local Dynamic Skill Loader."""
import pytest
from pathlib import Path
from app.skills.loader import SkillLoader, LocalSkill
from app.config import get_settings
def test_skill_loader_discovery():
"""Verify SkillLoader discovers all phase skills under app/skills/."""
settings = get_settings()
loader = SkillLoader(settings.SKILLS_DIR)
skills = loader.load_skills()
assert len(skills) >= 5
assert "source_discovery" in skills
assert "requirements_discovery" in skills
assert "architecture_design" in skills
assert "validation_rules" in skills
assert "packaging_guide" in skills
def test_skill_loader_by_phase():
"""Verify filtering skills by phase."""
settings = get_settings()
loader = SkillLoader(settings.SKILLS_DIR)
loader.load_skills()
src_discover_skills = loader.get_skills_by_phase("source_discover")
assert len(src_discover_skills) == 1
assert src_discover_skills[0].name == "source_discovery"
discover_skills = loader.get_skills_by_phase("discover")
assert len(discover_skills) == 1
assert discover_skills[0].name == "requirements_discovery"
design_skills = loader.get_skills_by_phase("design")
assert len(design_skills) == 1
assert design_skills[0].name == "architecture_design"
def test_format_skills_for_prompt():
"""Verify formatting skill content into prompt text."""
settings = get_settings()
loader = SkillLoader(settings.SKILLS_DIR)
loader.load_skills()
prompt = loader.format_skills_for_prompt("discover")
assert "## Injected Skill Instructions" in prompt
assert "requirements_discovery" in prompt

View File

@@ -3,7 +3,7 @@
import hashlib
from pathlib import Path
from app.agent_skills import discover_skills
from app.skills.agent_skills import discover_skills
VENDOR_DIR = Path(__file__).parents[1] / "skills" / "vendor" / "google"
PINNED_COMMIT = "81a31a6e4300425598a69fd96c8e843c947ae308"

View File

@@ -0,0 +1,18 @@
from pathlib import Path
import yaml
from app.workflows.deliverables import PHASE_ORDER, deliverables_in
ROOT = Path(__file__).parents[1]
def test_workflow_manifest_matches_the_dependency_graph():
manifest = yaml.safe_load((ROOT / "workflow.yaml").read_text(encoding="utf-8"))
declared = {phase["id"]: phase["deliverables"] for phase in manifest["phases"]}
assert declared == {
phase.value: [kind.value for kind in deliverables_in(phase)]
for phase in PHASE_ORDER
}

View File

@@ -85,6 +85,23 @@ async def test_validate_reports_errors():
assert "missing" in result.message
@needs_terraform
async def test_validate_runs_in_the_directory_holding_the_files():
broken = {"terraform/main.tf": 'output "x" {\n value = var.missing\n}\n'}
result = await TerraformRunner().validate(broken)
assert result.message.startswith("terraform validate failed")
async def test_files_across_directories_are_refused():
result = await TerraformRunner().validate(
{"terraform/main.tf": "", "modules/net/main.tf": ""}
)
assert result.message.startswith("Refused: put all files in one directory")
async def test_unsafe_file_names_are_refused():
runner = TerraformRunner()

View File

@@ -10,8 +10,8 @@ import uvicorn
from mcp.server.fastmcp import FastMCP
from mcp.types import ToolAnnotations
from app.execution.tools import gate_cloud_read
from app.providers import McpServerSpec
from app.tools.execution import gate_cloud_read
from app.tools.mcp import (
GoogleAdcAuth,
McpCatalog,