This commit is contained in:
@@ -8,15 +8,14 @@ import pytest
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from app.a2a.server import create_app
|
||||
from app.agents.fake_runtime import Call, FakeAgentRuntime
|
||||
from app.agents.roles import load_roles
|
||||
from app.assembly import Assembly
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from app.adk.roles import load_roles
|
||||
from app.agent import Assembly, ExecutionEngine
|
||||
from app.config import get_settings
|
||||
from app.execution.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
|
||||
from app.execution.engine import ExecutionEngine
|
||||
from app.execution.recorder import InMemoryRecorder
|
||||
from app.main import create_app
|
||||
from app.providers import load_provider
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
|
||||
|
||||
AGENTS_DIR = Path(__file__).parents[2] / "agents"
|
||||
K = DeliverableKind
|
||||
|
||||
@@ -11,13 +11,13 @@ from google.genai import types
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from pydantic import Field
|
||||
|
||||
from app.agents.adk_runtime import AdkAgentRuntime
|
||||
from app.agents.roles import load_roles
|
||||
from app.execution.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
||||
from app.execution.engine import ExecutionEngine
|
||||
from app.execution.recorder import InMemoryRecorder
|
||||
from app.execution.toolkit import make_tool_factory
|
||||
from app.adk.adk_runtime import AdkAgentRuntime
|
||||
from app.adk.roles import load_roles
|
||||
from app.agent import ExecutionEngine
|
||||
from app.providers import load_provider
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.tools.toolkit import make_tool_factory
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
||||
|
||||
AGENTS_DIR = Path(__file__).parents[2] / "agents"
|
||||
|
||||
@@ -47,7 +47,7 @@ def requirements_markdown() -> str:
|
||||
|
||||
|
||||
def task_context(request: LlmRequest) -> dict:
|
||||
text = request.contents[0].parts[0].text
|
||||
text = (request.contents[0].parts or [])[0].text or ""
|
||||
return json.loads(text.split("```json\n", 1)[1].rsplit("\n```", 1)[0])
|
||||
|
||||
|
||||
|
||||
@@ -3,16 +3,18 @@
|
||||
from collections.abc import AsyncGenerator
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from google.adk.models.base_llm import BaseLlm
|
||||
from google.adk.models.google_llm import Gemini
|
||||
from google.adk.models.lite_llm import LiteLlm
|
||||
from google.adk.models.llm_request import LlmRequest
|
||||
from google.adk.models.llm_response import LlmResponse
|
||||
from google.genai import types
|
||||
from pydantic import Field
|
||||
|
||||
from app.agents.adk_runtime import AdkAgentRuntime, default_model_factory
|
||||
from app.agents.runtime import (
|
||||
from app.adk.adk_runtime import AdkAgentRuntime, default_model_factory
|
||||
from app.adk.runtime import (
|
||||
AgentTask,
|
||||
BudgetExhausted,
|
||||
NeedsInput,
|
||||
@@ -197,12 +199,120 @@ async def test_budget_counts_every_tool_call_and_stops_the_run():
|
||||
assert len(recorder.calls) == 2
|
||||
|
||||
|
||||
async def test_model_reply_without_a_turn_ending_tool_is_stopped():
|
||||
model = ScriptedLlm(script=[[text("I need the RTO before I can continue.")]])
|
||||
async def test_model_reply_without_a_turn_ending_tool_is_stopped_after_nudges():
|
||||
model = ScriptedLlm(
|
||||
script=[[text("I need the RTO before I can continue.")], [text("")], [text("")]]
|
||||
)
|
||||
|
||||
outcome = await runtime_with(model).run(ROLE, TASK, [])
|
||||
|
||||
assert outcome == Stopped("I need the RTO before I can continue.")
|
||||
assert len(model.requests) == 3
|
||||
assert "without calling a tool that ends your turn" in (
|
||||
model.requests[1].contents[-1].parts[0].text
|
||||
)
|
||||
|
||||
|
||||
async def test_a_nudged_agent_that_then_ends_its_turn_succeeds_in_one_session():
|
||||
recorder = Recorder()
|
||||
model = ScriptedLlm(script=[[text("")], [call("delegate", note="go")]])
|
||||
|
||||
outcome = await runtime_with(model).run(
|
||||
RoleSpec("orchestrator", "Coordinate.", budget=5),
|
||||
TASK,
|
||||
[recorder.tool("delegate", TurnEnded("delegate"))],
|
||||
)
|
||||
|
||||
assert outcome == TurnEnded("delegate")
|
||||
assert "without calling a tool" in model.requests[1].contents[-1].parts[0].text
|
||||
|
||||
|
||||
class TimingOutLlm(ScriptedLlm):
|
||||
"""Times out on the first request, then follows its script."""
|
||||
|
||||
timed_out: bool = False
|
||||
|
||||
async def generate_content_async(
|
||||
self, llm_request: LlmRequest, stream: bool = False
|
||||
) -> AsyncGenerator[LlmResponse, None]:
|
||||
if not self.timed_out:
|
||||
self.timed_out = True
|
||||
raise httpx.ReadTimeout("no response")
|
||||
async for response in super().generate_content_async(llm_request, stream):
|
||||
yield response
|
||||
|
||||
|
||||
async def test_a_timed_out_model_request_is_retried_in_the_same_session():
|
||||
recorder = Recorder()
|
||||
model = TimingOutLlm(script=[[call("delegate", note="go")]])
|
||||
|
||||
outcome = await runtime_with(model).run(
|
||||
RoleSpec("orchestrator", "Coordinate.", budget=5),
|
||||
TASK,
|
||||
[recorder.tool("delegate", TurnEnded("delegate"))],
|
||||
)
|
||||
|
||||
assert outcome == TurnEnded("delegate")
|
||||
assert "timed out" in model.requests[0].contents[-1].parts[0].text
|
||||
|
||||
|
||||
async def test_model_timeout_is_set_on_every_request():
|
||||
model = ScriptedLlm(script=[[call("delegate")]])
|
||||
runtime = AdkAgentRuntime(
|
||||
skill_dirs=SKILLS, model_factory=lambda role: model, model_timeout_seconds=90
|
||||
)
|
||||
|
||||
await runtime.run(
|
||||
RoleSpec("orchestrator", "Coordinate.", budget=5),
|
||||
TASK,
|
||||
[Recorder().tool("delegate", TurnEnded("delegate"))],
|
||||
)
|
||||
|
||||
assert model.requests[0].config.http_options.timeout == 90_000
|
||||
|
||||
|
||||
class MeteredLlm(ScriptedLlm):
|
||||
async def generate_content_async(
|
||||
self, llm_request: LlmRequest, stream: bool = False
|
||||
) -> AsyncGenerator[LlmResponse, None]:
|
||||
async for response in super().generate_content_async(llm_request, stream):
|
||||
response.usage_metadata = types.GenerateContentResponseUsageMetadata(
|
||||
prompt_token_count=100,
|
||||
cached_content_token_count=40,
|
||||
candidates_token_count=10,
|
||||
thoughts_token_count=5,
|
||||
total_token_count=115,
|
||||
)
|
||||
yield response
|
||||
|
||||
|
||||
async def test_token_usage_is_summed_per_role_and_model():
|
||||
model = MeteredLlm(script=[[text("")], [call("delegate")]])
|
||||
role = RoleSpec("orchestrator", "Coordinate.", budget=5, model="gemini/x")
|
||||
runtime = runtime_with(model)
|
||||
|
||||
await runtime.run(role, TASK, [Recorder().tool("delegate", TurnEnded("delegate"))])
|
||||
|
||||
usage = runtime.usage[("orchestrator", "gemini/x")]
|
||||
assert (usage.requests, usage.prompt, usage.cached, usage.output) == (
|
||||
2,
|
||||
200,
|
||||
80,
|
||||
20,
|
||||
)
|
||||
assert (usage.thinking, usage.total) == (10, 230)
|
||||
|
||||
|
||||
async def test_task_message_names_the_roles_skills():
|
||||
model = ScriptedLlm(script=[[text("done")]] * 3)
|
||||
|
||||
await runtime_with(model).run(ROLE, TASK, [])
|
||||
|
||||
first = model.requests[0].contents[0].parts[0].text
|
||||
assert (
|
||||
"Skills available to you (load with load_skill): google-cloud-solution-architecture."
|
||||
in first
|
||||
)
|
||||
|
||||
|
||||
async def test_needs_input_outcome_passes_through():
|
||||
@@ -218,7 +328,9 @@ async def test_needs_input_outcome_passes_through():
|
||||
|
||||
async def test_each_run_starts_with_a_fresh_session():
|
||||
model = ScriptedLlm(script=[[text("first")], [text("second")]])
|
||||
runtime = runtime_with(model)
|
||||
runtime = AdkAgentRuntime(
|
||||
skill_dirs=SKILLS, model_factory=lambda role: model, max_nudges=0
|
||||
)
|
||||
|
||||
await runtime.run(ROLE, TASK, [])
|
||||
await runtime.run(ROLE, TASK, [])
|
||||
@@ -226,13 +338,19 @@ async def test_each_run_starts_with_a_fresh_session():
|
||||
assert len(model.requests[1].contents) == 1
|
||||
|
||||
|
||||
def test_default_model_factory_wraps_the_role_model_in_litellm():
|
||||
model = default_model_factory(
|
||||
RoleSpec("diagrammer", "", budget=1, model="gemini/gemini-flash-latest")
|
||||
def test_default_model_factory_uses_native_gemini_and_litellm_for_others():
|
||||
gemini = default_model_factory(
|
||||
RoleSpec("diagrammer", "", budget=1, model="gemini/gemini-3.8-flash")
|
||||
)
|
||||
other = default_model_factory(
|
||||
RoleSpec("diagrammer", "", budget=1, model="anthropic/claude-sonnet-5")
|
||||
)
|
||||
|
||||
assert isinstance(model, LiteLlm)
|
||||
assert model.model == "gemini/gemini-flash-latest"
|
||||
assert isinstance(gemini, Gemini)
|
||||
assert gemini.model == "gemini-3.8-flash"
|
||||
assert gemini.retry_options.attempts == 5
|
||||
assert isinstance(other, LiteLlm)
|
||||
assert other.model == "anthropic/claude-sonnet-5"
|
||||
with pytest.raises(ValueError, match="model"):
|
||||
default_model_factory(RoleSpec("diagrammer", "", budget=1))
|
||||
|
||||
@@ -245,7 +363,7 @@ def test_unknown_skill_is_rejected_when_the_runtime_is_built():
|
||||
)
|
||||
|
||||
|
||||
def _function_responses(request: LlmRequest) -> dict[str, dict]:
|
||||
def _function_responses(request: LlmRequest) -> dict:
|
||||
responses = {}
|
||||
for content in request.contents:
|
||||
for part in content.parts or []:
|
||||
|
||||
@@ -4,9 +4,9 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.agents.roles import PILLAR_ADVISOR, RoleConfigError, load_roles
|
||||
from app.execution.deliverables import DEPENDENCY_GRAPH
|
||||
from app.adk.roles import PILLAR_ADVISOR, RoleConfigError, load_roles
|
||||
from app.providers import load_provider
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH
|
||||
|
||||
AGENTS_DIR = Path(__file__).parents[2] / "agents"
|
||||
|
||||
@@ -34,7 +34,8 @@ def test_roles_take_skills_mcp_servers_and_models_from_the_bundle(bundle):
|
||||
assert roles["product-selector"].mcp_servers == ("developer_knowledge",)
|
||||
assert "asset_inventory" in roles["requirements-analyst"].mcp_servers
|
||||
assert roles["diagrammer"].model == bundle.models.for_role("diagrammer")
|
||||
assert roles["orchestrator"].model == bundle.models.default
|
||||
assert roles["orchestrator"].model == bundle.models.for_role("orchestrator")
|
||||
assert roles["decomposer"].model == bundle.models.default
|
||||
|
||||
|
||||
def test_roles_name_only_the_local_tools_they_need(bundle):
|
||||
|
||||
0
tests/eval/__init__.py
Normal file
0
tests/eval/__init__.py
Normal file
183
tests/eval/test_scoring.py
Normal file
183
tests/eval/test_scoring.py
Normal file
@@ -0,0 +1,183 @@
|
||||
"""Scoring a finished Execution against a benchmark case, without models or network."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.adk.runtime import ToolResult
|
||||
from app.agent import Interaction
|
||||
from eval.scoring import CaseSpec, score_execution, template_headings
|
||||
|
||||
TEMPLATE = (
|
||||
Path(__file__).parents[2]
|
||||
/ "skills/vendor/google/google-cloud-solution-architecture/assets/output-template.md"
|
||||
)
|
||||
HEADINGS = template_headings(TEMPLATE)
|
||||
|
||||
CASE = CaseSpec(
|
||||
id="case-x",
|
||||
name="Webhook ingestion",
|
||||
workflow_request="Ingest signed webhooks durably and keep 30 days of raw events.",
|
||||
required_products=("Cloud Run", "Pub/Sub", "Cloud Storage"),
|
||||
scripted_answers=(),
|
||||
)
|
||||
|
||||
|
||||
def guide(headings=HEADINGS) -> str:
|
||||
return "\n".join(f"## {heading}\nContent." for heading in headings)
|
||||
|
||||
|
||||
def execution(status="completed", **overrides) -> Interaction:
|
||||
deliverables = {
|
||||
"requirements": {
|
||||
"content": "## Functional requirements\nAccept webhooks.\n## Current state\nNone.",
|
||||
"attachments": {},
|
||||
},
|
||||
"product_selection": {
|
||||
"content": (
|
||||
"## Products and features mapping\n"
|
||||
"| Component | Product | Justification | Alternatives | Pros and cons |\n"
|
||||
"| Ingress | Cloud Run | [docs](https://cloud.google.com/run/docs) | GKE | "
|
||||
"**Pros**: control **Cons**: toil |\n"
|
||||
"| Queue | Pub/Sub | ... | Cloud Tasks | ... |\n| Archive | Cloud Storage | ... |"
|
||||
),
|
||||
"attachments": {},
|
||||
},
|
||||
"architecture_diagram": {
|
||||
"content": "diagram",
|
||||
"attachments": {
|
||||
"architecture.mmd": "flowchart TD\n A[Client] --> B[Run]\n"
|
||||
},
|
||||
},
|
||||
"deployment_guidance": {
|
||||
"content": "guide",
|
||||
"attachments": {
|
||||
"terraform/main.tf": "resource {}",
|
||||
"terraform/variables.tf": "",
|
||||
},
|
||||
},
|
||||
"solution_guide": {"content": guide(), "attachments": {}},
|
||||
}
|
||||
for kind, value in overrides.items():
|
||||
deliverables[kind] = value
|
||||
return Interaction(
|
||||
"exec-1", status, None, {"deliverables": deliverables, "warnings": []}
|
||||
)
|
||||
|
||||
|
||||
async def passing_terraform(files) -> ToolResult:
|
||||
return ToolResult("terraform validate passed:\nSuccess!")
|
||||
|
||||
|
||||
async def failing_terraform(files) -> ToolResult:
|
||||
return ToolResult("terraform validate failed:\nError: Unsupported argument")
|
||||
|
||||
|
||||
def check(result, name):
|
||||
return next(c for c in result.checks if c.name == name)
|
||||
|
||||
|
||||
def test_template_headings_skip_the_title_and_placeholders():
|
||||
assert HEADINGS[0] == "1. Executive summary and workload overview"
|
||||
assert "4.1. Google Cloud products and features mapping" in HEADINGS
|
||||
assert not any("[" in heading for heading in HEADINGS)
|
||||
assert len(HEADINGS) >= 20
|
||||
|
||||
|
||||
async def test_a_complete_execution_passes_every_check():
|
||||
result = await score_execution(CASE, execution(), HEADINGS, passing_terraform)
|
||||
|
||||
assert result.passed, [c for c in result.checks if not c.passed]
|
||||
assert result.score == 1.0
|
||||
|
||||
|
||||
async def test_missing_template_sections_are_named():
|
||||
partial = {"content": guide(HEADINGS[:-2]), "attachments": {}}
|
||||
|
||||
result = await score_execution(
|
||||
CASE, execution(solution_guide=partial), HEADINGS, passing_terraform
|
||||
)
|
||||
|
||||
detail = check(result, "guide_follows_template").detail
|
||||
assert not check(result, "guide_follows_template").passed
|
||||
assert HEADINGS[-1] in detail
|
||||
|
||||
|
||||
async def test_requirements_naming_products_not_in_the_request_breaks_phase_separation():
|
||||
leaky = {"content": "## Functional requirements\nUse Cloud Run.", "attachments": {}}
|
||||
|
||||
result = await score_execution(
|
||||
CASE, execution(requirements=leaky), HEADINGS, passing_terraform
|
||||
)
|
||||
|
||||
assert not check(result, "requirements_name_no_products").passed
|
||||
assert "Cloud Run" in check(result, "requirements_name_no_products").detail
|
||||
|
||||
|
||||
async def test_products_named_by_the_user_may_appear_in_requirements():
|
||||
case = CaseSpec(
|
||||
**{**CASE.__dict__, "workflow_request": "Use Cloud Run for webhooks."}
|
||||
)
|
||||
named = {
|
||||
"content": "## Functional requirements\nRuns on Cloud Run.",
|
||||
"attachments": {},
|
||||
}
|
||||
|
||||
result = await score_execution(
|
||||
case, execution(requirements=named), HEADINGS, passing_terraform
|
||||
)
|
||||
|
||||
assert check(result, "requirements_name_no_products").passed
|
||||
|
||||
|
||||
async def test_selection_without_alternatives_or_citations_fails_those_checks():
|
||||
bare = {
|
||||
"content": "## Products and features mapping\nCloud Run, Pub/Sub, Cloud Storage.",
|
||||
"attachments": {},
|
||||
}
|
||||
|
||||
result = await score_execution(
|
||||
CASE, execution(product_selection=bare), HEADINGS, passing_terraform
|
||||
)
|
||||
|
||||
assert not check(result, "selection_has_alternatives").passed
|
||||
assert not check(result, "selection_cites_documentation").passed
|
||||
assert check(result, "required_products_selected").passed
|
||||
|
||||
|
||||
async def test_invalid_terraform_and_diagram_fail_and_warnings_fail_completion():
|
||||
diagram = {"content": "d", "attachments": {"architecture.mmd": "A --> B"}}
|
||||
|
||||
result = await score_execution(
|
||||
CASE,
|
||||
execution("completed_with_warnings", architecture_diagram=diagram),
|
||||
HEADINGS,
|
||||
failing_terraform,
|
||||
)
|
||||
|
||||
assert not check(result, "completed").passed
|
||||
assert not check(result, "terraform_validates").passed
|
||||
assert not check(result, "diagram_is_valid_mermaid").passed
|
||||
assert not result.passed
|
||||
|
||||
|
||||
async def test_an_execution_that_never_produced_deliverables_scores_zero_safely():
|
||||
empty = Interaction("exec-2", "completed_with_warnings", None, {"deliverables": {}})
|
||||
|
||||
result = await score_execution(CASE, empty, HEADINGS, passing_terraform)
|
||||
|
||||
assert result.score == 0.0
|
||||
assert not result.passed
|
||||
|
||||
|
||||
def test_benchmark_cases_load_with_valid_answer_patterns():
|
||||
import re
|
||||
|
||||
from eval.scoring import load_cases
|
||||
|
||||
cases = load_cases(Path(__file__).parents[2] / "eval/datasets/solution_cases.json")
|
||||
|
||||
assert len(cases) >= 2
|
||||
for case in cases:
|
||||
assert case.required_products
|
||||
for pattern, answer in case.scripted_answers:
|
||||
re.compile(pattern)
|
||||
assert answer
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import pytest
|
||||
|
||||
from app.execution.deliverables import (
|
||||
from app.workflows.deliverables import (
|
||||
DEPENDENCY_GRAPH,
|
||||
PHASE_ORDER,
|
||||
DeliverableKind,
|
||||
|
||||
@@ -9,11 +9,11 @@ import pytest
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
|
||||
from app.agents.fake_runtime import Call, FakeAgentRuntime
|
||||
from app.agents.runtime import RoleSpec, ToolResult, ToolSpec
|
||||
from app.execution.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
|
||||
from app.execution.engine import ExecutionEngine
|
||||
from app.execution.recorder import InMemoryRecorder
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
|
||||
from app.agent import ExecutionEngine
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind, PhaseId
|
||||
|
||||
K = DeliverableKind
|
||||
P = PhaseId
|
||||
@@ -661,3 +661,28 @@ async def test_a_failing_specialist_is_relayed_to_the_user_as_a_question(tmp_pat
|
||||
assert question["from"] == "requirements"
|
||||
assert "model API unavailable" in question["question"]
|
||||
assert result.state["ledger"]["deliverables"]["requirements"]["status"] == "pending"
|
||||
|
||||
|
||||
async def test_pre_approved_questions_use_matching_scripted_answers(tmp_path):
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
"orchestrator": [
|
||||
[ask_user("What is the RTO?", "Which region?")],
|
||||
[ask_user("Anything else?")],
|
||||
]
|
||||
}
|
||||
)
|
||||
engine = make_engine(tmp_path, runtime, max_orchestrator_turns=2)
|
||||
|
||||
result = await engine.start(
|
||||
"Ingestion",
|
||||
pre_approved=True,
|
||||
scripted_answers=[(r"\bRTO\b", "One hour."), ("region", "europe-west2")],
|
||||
)
|
||||
|
||||
first_reply = result.state["conversation"][2]["text"]
|
||||
assert "Q: What is the RTO?\nA: One hour." in first_reply
|
||||
assert "Q: Which region?\nA: europe-west2" in first_reply
|
||||
assert "No user is available" not in first_reply
|
||||
second_reply = result.state["conversation"][4]["text"]
|
||||
assert second_reply.startswith("No user is available")
|
||||
|
||||
72
tests/execution/test_file_arguments.py
Normal file
72
tests/execution/test_file_arguments.py
Normal file
@@ -0,0 +1,72 @@
|
||||
"""Files cross tool calls as a list of {path, content}: models rewrite object keys."""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.adk.runtime import FILES_SCHEMA, Submitted, files_from_argument
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.tools.local import TerraformRunner, local_tools
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
GUIDANCE = (
|
||||
"# Deployment prerequisites\n\nx\n\n# Step-by-step deployment instructions\n\ny\n"
|
||||
)
|
||||
|
||||
|
||||
def test_files_are_offered_as_a_list_of_path_and_content():
|
||||
assert FILES_SCHEMA["type"] == "array"
|
||||
assert FILES_SCHEMA["items"]["required"] == ["path", "content"]
|
||||
|
||||
|
||||
def test_file_arguments_accept_lists_and_maps():
|
||||
as_list = [{"path": "terraform/main.tf", "content": "a"}]
|
||||
|
||||
assert files_from_argument(as_list) == {"terraform/main.tf": "a"}
|
||||
assert files_from_argument({"terraform/main.tf": "a"}) == {"terraform/main.tf": "a"}
|
||||
assert files_from_argument(None) == {}
|
||||
with pytest.raises(TypeError):
|
||||
files_from_argument(["terraform/main.tf"])
|
||||
|
||||
|
||||
async def test_submission_takes_attachments_as_a_list():
|
||||
submit = specialist_tools(
|
||||
DeliverableKind.DEPLOYMENT_GUIDANCE, extra=[], allow_cloud_read=False
|
||||
)[0]
|
||||
|
||||
result = await submit.handler(
|
||||
content=GUIDANCE,
|
||||
attachments=[
|
||||
{"path": "terraform/main.tf", "content": "resource {}"},
|
||||
{"path": "terraform/variables.tf", "content": ""},
|
||||
],
|
||||
)
|
||||
|
||||
assert result.outcome == Submitted(
|
||||
GUIDANCE, {"terraform/main.tf": "resource {}", "terraform/variables.tf": ""}
|
||||
)
|
||||
|
||||
|
||||
async def test_malformed_attachments_are_refused_to_the_agent():
|
||||
submit = specialist_tools(
|
||||
DeliverableKind.DEPLOYMENT_GUIDANCE, extra=[], allow_cloud_read=False
|
||||
)[0]
|
||||
|
||||
result = await submit.handler(content=GUIDANCE, attachments=["terraform/main.tf"])
|
||||
|
||||
assert result.outcome is None
|
||||
assert result.message.startswith("Refused: attachments")
|
||||
|
||||
|
||||
async def test_terraform_tools_take_files_as_a_list():
|
||||
received = {}
|
||||
|
||||
class Runner(TerraformRunner):
|
||||
async def validate(self, files):
|
||||
received.update(files)
|
||||
return await super().validate({})
|
||||
|
||||
(validate,) = local_tools(["terraform_validate"], Runner())
|
||||
|
||||
await validate.handler(files=[{"path": "terraform/main.tf", "content": "x"}])
|
||||
|
||||
assert validate.parameters["properties"]["files"]["type"] == "array"
|
||||
assert received == {"terraform/main.tf": "x"}
|
||||
@@ -2,14 +2,14 @@
|
||||
|
||||
import pytest
|
||||
|
||||
from app.execution.deliverables import DeliverableKind as K
|
||||
from app.execution.deliverables import PhaseId as P
|
||||
from app.execution.ledger import (
|
||||
from app.states.ledger import (
|
||||
DeliverableStatus,
|
||||
Ledger,
|
||||
LedgerRefusal,
|
||||
PhaseStatus,
|
||||
)
|
||||
from app.workflows.deliverables import DeliverableKind as K
|
||||
from app.workflows.deliverables import PhaseId as P
|
||||
|
||||
|
||||
def produce(ledger: Ledger, *kinds: K) -> None:
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
"""The design advisor's fan-out to one advisor per architecture-framework pillar."""
|
||||
|
||||
from app.agents.fake_runtime import Call, FakeAgentRuntime
|
||||
from app.agents.roles import PILLAR_ADVISOR
|
||||
from app.agents.runtime import RoleSpec, ToolResult, ToolSpec
|
||||
from app.execution.pillars import CONSULT_TOOL, pillar_consultation_tool
|
||||
from app.execution.toolkit import make_tool_factory
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from app.adk.roles import PILLAR_ADVISOR
|
||||
from app.adk.runtime import RoleSpec, ToolResult, ToolSpec
|
||||
from app.tools.pillars import CONSULT_TOOL, pillar_consultation_tool
|
||||
from app.tools.toolkit import make_tool_factory
|
||||
|
||||
PILLARS = {
|
||||
"security": "waf-security",
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
import json
|
||||
|
||||
from app.execution.engine import Interaction
|
||||
from app.execution.presentation import describe_prompt, to_generate_response
|
||||
from app.agent import Interaction
|
||||
from app.presentation import describe_prompt, to_generate_response
|
||||
|
||||
|
||||
def test_approval_prompt_explains_the_structured_reply():
|
||||
|
||||
@@ -6,11 +6,11 @@ import pytest
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
|
||||
from app.checkpointing import open_checkpointer
|
||||
from app.config import get_settings
|
||||
from app.database import DatabaseManager
|
||||
from app.execution.checkpointing import open_checkpointer
|
||||
from app.execution.ledger import LedgerEvent
|
||||
from app.execution.recorder import DatabaseRecorder
|
||||
from app.recorder import DatabaseRecorder
|
||||
from app.states.ledger import LedgerEvent
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -1,212 +0,0 @@
|
||||
"""Comprehensive test suite for Google ADK Multi-Agent Architecture & PostgreSQL Persistence."""
|
||||
|
||||
import pytest
|
||||
import uuid
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from app.adk.agents import (
|
||||
DiscoveryAgent,
|
||||
ArchitectureDesignAgent,
|
||||
ValidationReviewAgent,
|
||||
PackagingAgent,
|
||||
OrchestratorLoopAgent,
|
||||
build_adk_multi_agent_system,
|
||||
)
|
||||
from app.adk.artifacts import PostgresArtifactRepository
|
||||
from app.adk.evaluation import ADKEvaluator
|
||||
from app.adk.runners import ADKAgentRunner
|
||||
from app.adk.sessions import PostgresSessionService
|
||||
from app.adk.tools import ADK_TOOLS
|
||||
from app.adk.workflows import create_gcp_adk_workflow
|
||||
from app.agent import app
|
||||
from app.config import get_settings
|
||||
from app.database import get_db_manager
|
||||
from app.skills.loader import SkillLoader
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def skill_loader():
|
||||
settings = get_settings()
|
||||
loader = SkillLoader(settings.SKILLS_DIR)
|
||||
loader.load_skills()
|
||||
return loader
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_database_manager_operations():
|
||||
"""Verify PostgreSQL database manager schema initialization & CRUD operations."""
|
||||
db = get_db_manager()
|
||||
session_id = f"test-sess-{uuid.uuid4()}"
|
||||
execution_id = f"test-exec-{uuid.uuid4()}"
|
||||
|
||||
# 1. Save and retrieve session
|
||||
saved = db.save_session(session_id, "TestAgent", {"key": "value"})
|
||||
assert saved is True
|
||||
|
||||
session_data = db.get_session(session_id)
|
||||
assert session_data is not None
|
||||
assert session_data["agent_name"] == "TestAgent"
|
||||
assert session_data["state_data"]["key"] == "value"
|
||||
|
||||
# 2. Save workflow execution
|
||||
saved_exec = db.save_workflow_execution(
|
||||
execution_id=execution_id,
|
||||
session_id=session_id,
|
||||
status="completed",
|
||||
current_phase="package",
|
||||
loop_count=1,
|
||||
request_summary="Unit test workflow request",
|
||||
results={"passed": True},
|
||||
)
|
||||
assert saved_exec is True
|
||||
|
||||
# 3. Save artifact
|
||||
art_saved = db.save_artifact(
|
||||
artifact_id=f"art-{uuid.uuid4()}",
|
||||
execution_id=execution_id,
|
||||
artifact_type="markdown",
|
||||
file_path="docs/test.md",
|
||||
content="# Test Content",
|
||||
)
|
||||
assert art_saved is True
|
||||
|
||||
# 4. Record review log
|
||||
log_saved = db.record_orchestrator_log(
|
||||
log_id=f"log-{uuid.uuid4()}",
|
||||
execution_id=execution_id,
|
||||
iteration=1,
|
||||
review_status="APPROVED",
|
||||
reviewer_agent="ValidationReviewAgent",
|
||||
feedback="Passed all checks",
|
||||
)
|
||||
assert log_saved is True
|
||||
|
||||
|
||||
def test_adk_tools():
|
||||
"""Verify google.adk.tools wrappers."""
|
||||
assert len(ADK_TOOLS) == 6
|
||||
tool_names = [t.name for t in ADK_TOOLS]
|
||||
assert "validate_mermaid_diagram" in tool_names
|
||||
assert "validate_terraform_syntax" in tool_names
|
||||
assert "validate_repository_artifacts" in tool_names
|
||||
assert "developerknowledge_search_documents" in tool_names
|
||||
assert "developerknowledge_get_documents" in tool_names
|
||||
assert "developerknowledge_answer_query" in tool_names
|
||||
|
||||
|
||||
def test_adk_postgres_session_service():
|
||||
"""Verify PostgresSessionService creating, getting, and saving session state."""
|
||||
session_service = PostgresSessionService()
|
||||
session_id = f"adk-session-{uuid.uuid4()}"
|
||||
|
||||
sess = session_service.create_session("GCP_ADK_Agent", session_id=session_id)
|
||||
assert sess.session_id == session_id
|
||||
assert sess.agent_name == "GCP_ADK_Agent"
|
||||
|
||||
sess.state["counter"] = 42
|
||||
saved = session_service.save_session(sess)
|
||||
assert saved is True
|
||||
|
||||
fetched = session_service.get_session(session_id)
|
||||
assert fetched is not None
|
||||
assert fetched.state.get("counter") == 42
|
||||
|
||||
|
||||
def test_adk_orchestrator_loop_agent(skill_loader):
|
||||
"""Verify OrchestratorLoopAgent multi-agent loop review cycle."""
|
||||
orchestrator = build_adk_multi_agent_system(skill_loader, max_iterations=3)
|
||||
assert orchestrator.name == "OrchestratorLoopAgent"
|
||||
assert orchestrator.max_iterations == 3
|
||||
|
||||
initial_state = {
|
||||
"execution_id": str(uuid.uuid4()),
|
||||
"workflow_request": "Event-driven regional HTTP application",
|
||||
"target_dir": ".",
|
||||
"active_skills": [],
|
||||
}
|
||||
|
||||
final_state = orchestrator.execute(initial_state)
|
||||
assert final_state.get("validation_passed") is True
|
||||
assert final_state.get("loop_completed_successfully") is True
|
||||
assert len(final_state.get("active_skills", [])) == 5
|
||||
|
||||
|
||||
def test_adk_workflow_composition(skill_loader):
|
||||
"""Verify google.adk.workflows Workflow composition."""
|
||||
workflow = create_gcp_adk_workflow(skill_loader, max_iterations=2)
|
||||
assert workflow.name == "GCP_Solution_Architecture_Workflow"
|
||||
assert len(workflow.steps) == 1
|
||||
|
||||
initial_state = {
|
||||
"execution_id": str(uuid.uuid4()),
|
||||
"workflow_request": "Event-driven architecture test",
|
||||
"target_dir": ".",
|
||||
"active_skills": [],
|
||||
}
|
||||
|
||||
result = workflow.execute(initial_state)
|
||||
assert result.get("validation_passed") is True
|
||||
|
||||
|
||||
def test_adk_agent_runner():
|
||||
"""Verify ADKAgentRunner execution and artifact persistence."""
|
||||
runner = ADKAgentRunner()
|
||||
session_id = f"runner-session-{uuid.uuid4()}"
|
||||
|
||||
res = runner.run_execution(
|
||||
session_id=session_id,
|
||||
request_summary="High scale ingestion workflow",
|
||||
target_dir=".",
|
||||
)
|
||||
|
||||
assert res["status"] == "success"
|
||||
assert res["validation_passed"] is True
|
||||
assert "solution_guide" in res["artifacts"]
|
||||
|
||||
# Verify session persists in Postgres DB
|
||||
session_service = PostgresSessionService()
|
||||
sess = session_service.get_session(session_id)
|
||||
assert sess is not None
|
||||
|
||||
|
||||
def test_adk_evaluator():
|
||||
"""Verify ADKEvaluator executing benchmark cases."""
|
||||
evaluator = ADKEvaluator()
|
||||
case = {
|
||||
"id": "case-test",
|
||||
"workflow_request": "Event-driven regional HTTP application",
|
||||
"rubric": {
|
||||
"required_products": ["Cloud Run", "Pub/Sub", "Cloud Storage"],
|
||||
"requires_mermaid": True,
|
||||
"requires_terraform": True,
|
||||
"requires_guide_sections": ["Functional requirements", "Selected products"],
|
||||
},
|
||||
}
|
||||
|
||||
res = evaluator.evaluate_benchmark_case(case)
|
||||
assert res["passed"] is True
|
||||
assert res["percentage"] >= 80.0
|
||||
|
||||
|
||||
def test_starlette_session_api_endpoint(client):
|
||||
"""Verify Starlette API GET /sessions/{session_id} route."""
|
||||
session_id = f"api-sess-{uuid.uuid4()}"
|
||||
|
||||
# First run workflow to populate session
|
||||
payload = {
|
||||
"session_id": session_id,
|
||||
"request": "Test session retrieval route",
|
||||
}
|
||||
gen_resp = client.post("/generate", json=payload)
|
||||
assert gen_resp.status_code == 200
|
||||
|
||||
# Query session
|
||||
sess_resp = client.get(f"/sessions/{session_id}")
|
||||
assert sess_resp.status_code == 200
|
||||
sess_data = sess_resp.json()
|
||||
assert sess_data["session_id"] == session_id
|
||||
assert sess_data["agent_name"] == "OrchestratorLoopAgent"
|
||||
@@ -1,52 +0,0 @@
|
||||
"""Starlette REST API integration tests using TestClient."""
|
||||
|
||||
import pytest
|
||||
from starlette.testclient import TestClient
|
||||
from app.agent import app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_health_endpoint(client):
|
||||
"""GET /health returns healthy status."""
|
||||
response = client.get("/health")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "healthy"
|
||||
assert "gcp_solution_architecture_agent" in data["agent"]
|
||||
|
||||
|
||||
def test_card_endpoint(client):
|
||||
"""GET /card returns agent capability card."""
|
||||
response = client.get("/card")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["name"] == "gcp_solution_architecture_agent"
|
||||
assert "discover" in data["capabilities"]["phases"]
|
||||
|
||||
|
||||
def test_generate_endpoint(client):
|
||||
"""POST /generate executes full agent workflow."""
|
||||
payload = {
|
||||
"request": "Build an event-driven regional ingestion service",
|
||||
"target_dir": ".",
|
||||
}
|
||||
response = client.post("/generate", json=payload)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "success"
|
||||
assert data["validation_passed"] is True
|
||||
assert "artifacts" in data
|
||||
assert "solution_guide" in data["artifacts"]
|
||||
|
||||
|
||||
def test_validate_endpoint(client):
|
||||
"""POST /validate executes pre-deployment validation checks."""
|
||||
payload = {"target_dir": "."}
|
||||
response = client.post("/validate", json=payload)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["valid"] is True
|
||||
@@ -4,7 +4,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.agent_skills import SkillFormatError, discover_skills, read_skill
|
||||
from app.skills.agent_skills import SkillFormatError, discover_skills, read_skill
|
||||
|
||||
|
||||
def write_skill(
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import pathlib
|
||||
import re
|
||||
import unittest
|
||||
|
||||
ROOT = pathlib.Path(__file__).parents[1]
|
||||
@@ -46,7 +45,13 @@ class ArtifactTests(unittest.TestCase):
|
||||
|
||||
def test_guide_packages_outputs(self):
|
||||
guide = get_artifact_text("solution-architecture-guide.md")
|
||||
for heading in ("Requirements", "Architecture", "Terraform", "Validation", "Verification"):
|
||||
for heading in (
|
||||
"Requirements",
|
||||
"Architecture",
|
||||
"Terraform",
|
||||
"Validation",
|
||||
"Verification",
|
||||
):
|
||||
self.assertIn(heading, guide)
|
||||
|
||||
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
"""Building the engine from settings and the provider bundle."""
|
||||
|
||||
from app.agents.fake_runtime import Call, FakeAgentRuntime
|
||||
from app.assembly import open_assembly
|
||||
from typing import Any
|
||||
|
||||
from app.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from app.agent import open_assembly
|
||||
from app.config import Settings
|
||||
from app.execution.recorder import InMemoryRecorder
|
||||
from app.recorder import InMemoryRecorder
|
||||
|
||||
|
||||
def settings(tmp_path, **overrides) -> Settings:
|
||||
values = {
|
||||
values: dict[str, Any] = {
|
||||
"CHECKPOINTER": "memory",
|
||||
"MCP_ENABLED": False,
|
||||
"ORCHESTRATOR_BUDGET": 9,
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
"""Integration tests for LangGraph StateGraph execution."""
|
||||
|
||||
import pytest
|
||||
from app.workflows.gcp_architecture_graph import create_agent
|
||||
from app.skills.loader import SkillLoader
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
def test_agent_graph_execution():
|
||||
"""Verify execution of the 4-phase LangGraph StateGraph."""
|
||||
settings = get_settings()
|
||||
loader = SkillLoader(settings.SKILLS_DIR)
|
||||
loader.load_skills()
|
||||
|
||||
agent = create_agent(loader)
|
||||
initial_state = {
|
||||
"workflow_request": "Event-driven regional HTTP application",
|
||||
"target_dir": ".",
|
||||
"active_skills": [],
|
||||
}
|
||||
|
||||
final_state = agent.invoke(initial_state)
|
||||
|
||||
assert final_state.get("current_phase") == "package"
|
||||
assert final_state.get("source_discovery_doc") is not None
|
||||
assert final_state.get("source_mermaid_diagram") is not None
|
||||
assert final_state.get("requirements_doc") is not None
|
||||
assert final_state.get("architecture_doc") is not None
|
||||
assert final_state.get("mermaid_diagram") is not None
|
||||
assert final_state.get("terraform_code") is not None
|
||||
assert final_state.get("validation_results") is not None
|
||||
assert final_state.get("solution_guide") is not None
|
||||
assert final_state.get("validation_passed") is True
|
||||
assert len(final_state.get("active_skills", [])) == 5
|
||||
@@ -1,53 +0,0 @@
|
||||
"""Unit & Integration tests for Developer Knowledge MCP Client and Tools."""
|
||||
|
||||
import pytest
|
||||
from app.tools.mcp_developer_knowledge import (
|
||||
DeveloperKnowledgeMCPClient,
|
||||
developerknowledge_answer_query,
|
||||
developerknowledge_get_documents,
|
||||
developerknowledge_search_documents,
|
||||
get_mcp_client,
|
||||
)
|
||||
from app.adk.tools import mcp_search_tool, mcp_get_tool, mcp_answer_tool
|
||||
|
||||
|
||||
def test_mcp_client_search_documents():
|
||||
"""Verify DeveloperKnowledgeMCPClient search_documents method."""
|
||||
client = get_mcp_client()
|
||||
res = client.search_documents(query="Cloud Run", category="compute")
|
||||
assert res["source"] in ("live_mcp", "offline_fallback")
|
||||
assert "documents" in res or "data" in res
|
||||
|
||||
|
||||
def test_mcp_client_get_documents():
|
||||
"""Verify DeveloperKnowledgeMCPClient get_documents method."""
|
||||
client = get_mcp_client()
|
||||
res = client.get_documents(document_uri="https://cloud.google.com/run/docs/overview")
|
||||
assert res["source"] in ("live_mcp", "offline_fallback")
|
||||
assert "document" in res or "data" in res
|
||||
|
||||
|
||||
def test_mcp_client_answer_query():
|
||||
"""Verify DeveloperKnowledgeMCPClient answer_query method."""
|
||||
client = get_mcp_client()
|
||||
res = client.answer_query(query="Check Cloud Run release status")
|
||||
assert res["source"] in ("live_mcp", "offline_fallback")
|
||||
|
||||
|
||||
def test_langchain_mcp_tools():
|
||||
"""Verify LangChain @tool wrappers for Developer Knowledge MCP."""
|
||||
search_res = developerknowledge_search_documents.invoke({"query": "PubSub", "category": "messaging"})
|
||||
assert search_res is not None
|
||||
|
||||
get_res = developerknowledge_get_documents.invoke({"document_uri": "https://cloud.google.com/pubsub/docs"})
|
||||
assert get_res is not None
|
||||
|
||||
ans_res = developerknowledge_answer_query.invoke({"query": "What is the release status of Cloud Storage?"})
|
||||
assert ans_res is not None
|
||||
|
||||
|
||||
def test_adk_mcp_function_tools():
|
||||
"""Verify ADK FunctionTools for Developer Knowledge MCP."""
|
||||
assert mcp_search_tool.name == "developerknowledge_search_documents"
|
||||
assert mcp_get_tool.name == "developerknowledge_get_documents"
|
||||
assert mcp_answer_tool.name == "developerknowledge_answer_query"
|
||||
@@ -2,9 +2,21 @@ import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).parents[1]
|
||||
|
||||
|
||||
def test_artifact_validator():
|
||||
result = subprocess.run(["python3", "scripts/validate_artifacts.py"], cwd=ROOT, capture_output=True, text=True)
|
||||
result = subprocess.run(
|
||||
["python3", "scripts/validate_artifacts.py"],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
|
||||
def test_no_secret_values():
|
||||
assert "replace-with-project-id" in (ROOT / "terraform/terraform.tfvars.example").read_text()
|
||||
assert (
|
||||
"replace-with-project-id"
|
||||
in (ROOT / "terraform/terraform.tfvars.example").read_text()
|
||||
)
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
"""Unit tests for Local Dynamic Skill Loader."""
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from app.skills.loader import SkillLoader, LocalSkill
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
def test_skill_loader_discovery():
|
||||
"""Verify SkillLoader discovers all phase skills under app/skills/."""
|
||||
settings = get_settings()
|
||||
loader = SkillLoader(settings.SKILLS_DIR)
|
||||
skills = loader.load_skills()
|
||||
|
||||
assert len(skills) >= 5
|
||||
assert "source_discovery" in skills
|
||||
assert "requirements_discovery" in skills
|
||||
assert "architecture_design" in skills
|
||||
assert "validation_rules" in skills
|
||||
assert "packaging_guide" in skills
|
||||
|
||||
|
||||
def test_skill_loader_by_phase():
|
||||
"""Verify filtering skills by phase."""
|
||||
settings = get_settings()
|
||||
loader = SkillLoader(settings.SKILLS_DIR)
|
||||
loader.load_skills()
|
||||
|
||||
src_discover_skills = loader.get_skills_by_phase("source_discover")
|
||||
assert len(src_discover_skills) == 1
|
||||
assert src_discover_skills[0].name == "source_discovery"
|
||||
|
||||
discover_skills = loader.get_skills_by_phase("discover")
|
||||
assert len(discover_skills) == 1
|
||||
assert discover_skills[0].name == "requirements_discovery"
|
||||
|
||||
design_skills = loader.get_skills_by_phase("design")
|
||||
assert len(design_skills) == 1
|
||||
assert design_skills[0].name == "architecture_design"
|
||||
|
||||
|
||||
def test_format_skills_for_prompt():
|
||||
"""Verify formatting skill content into prompt text."""
|
||||
settings = get_settings()
|
||||
loader = SkillLoader(settings.SKILLS_DIR)
|
||||
loader.load_skills()
|
||||
|
||||
prompt = loader.format_skills_for_prompt("discover")
|
||||
assert "## Injected Skill Instructions" in prompt
|
||||
assert "requirements_discovery" in prompt
|
||||
@@ -3,7 +3,7 @@
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from app.agent_skills import discover_skills
|
||||
from app.skills.agent_skills import discover_skills
|
||||
|
||||
VENDOR_DIR = Path(__file__).parents[1] / "skills" / "vendor" / "google"
|
||||
PINNED_COMMIT = "81a31a6e4300425598a69fd96c8e843c947ae308"
|
||||
|
||||
18
tests/test_workflow_manifest.py
Normal file
18
tests/test_workflow_manifest.py
Normal file
@@ -0,0 +1,18 @@
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from app.workflows.deliverables import PHASE_ORDER, deliverables_in
|
||||
|
||||
ROOT = Path(__file__).parents[1]
|
||||
|
||||
|
||||
def test_workflow_manifest_matches_the_dependency_graph():
|
||||
manifest = yaml.safe_load((ROOT / "workflow.yaml").read_text(encoding="utf-8"))
|
||||
|
||||
declared = {phase["id"]: phase["deliverables"] for phase in manifest["phases"]}
|
||||
|
||||
assert declared == {
|
||||
phase.value: [kind.value for kind in deliverables_in(phase)]
|
||||
for phase in PHASE_ORDER
|
||||
}
|
||||
@@ -85,6 +85,23 @@ async def test_validate_reports_errors():
|
||||
assert "missing" in result.message
|
||||
|
||||
|
||||
@needs_terraform
|
||||
async def test_validate_runs_in_the_directory_holding_the_files():
|
||||
broken = {"terraform/main.tf": 'output "x" {\n value = var.missing\n}\n'}
|
||||
|
||||
result = await TerraformRunner().validate(broken)
|
||||
|
||||
assert result.message.startswith("terraform validate failed")
|
||||
|
||||
|
||||
async def test_files_across_directories_are_refused():
|
||||
result = await TerraformRunner().validate(
|
||||
{"terraform/main.tf": "", "modules/net/main.tf": ""}
|
||||
)
|
||||
|
||||
assert result.message.startswith("Refused: put all files in one directory")
|
||||
|
||||
|
||||
async def test_unsafe_file_names_are_refused():
|
||||
runner = TerraformRunner()
|
||||
|
||||
|
||||
@@ -10,8 +10,8 @@ import uvicorn
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from mcp.types import ToolAnnotations
|
||||
|
||||
from app.execution.tools import gate_cloud_read
|
||||
from app.providers import McpServerSpec
|
||||
from app.tools.execution import gate_cloud_read
|
||||
from app.tools.mcp import (
|
||||
GoogleAdcAuth,
|
||||
McpCatalog,
|
||||
|
||||
Reference in New Issue
Block a user