This commit is contained in:
BIN
app/__pycache__/__init__.cpython-312.pyc
Normal file
BIN
app/__pycache__/__init__.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/__pycache__/agent.cpython-312.pyc
Normal file
BIN
app/__pycache__/agent.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/__pycache__/card.cpython-312.pyc
Normal file
BIN
app/__pycache__/card.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/__pycache__/config.cpython-312.pyc
Normal file
BIN
app/__pycache__/config.cpython-312.pyc
Normal file
Binary file not shown.
50
app/agent.py
50
app/agent.py
@@ -1 +1,49 @@
|
||||
"""gcp_solution_architecture_agent: create_agent_app wiring + REST route handlers only."""
|
||||
"""gcp_solution_architecture_agent: Starlette application wiring & CLI entrypoint."""
|
||||
|
||||
import logging
|
||||
import click
|
||||
import uvicorn
|
||||
from starlette.applications import Starlette
|
||||
from starlette.routing import Route
|
||||
|
||||
from app.config import get_settings
|
||||
from app.workflows.routes import (
|
||||
get_card,
|
||||
health_check,
|
||||
run_workflow,
|
||||
validate_artifacts_route,
|
||||
)
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, settings.LOG_LEVEL.upper(), logging.INFO),
|
||||
format="%(asctime)s %(name)s %(levelname)s %(message)s",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
routes = [
|
||||
Route("/health", health_check, methods=["GET"]),
|
||||
Route("/card", get_card, methods=["GET"]),
|
||||
Route("/generate", run_workflow, methods=["POST"]),
|
||||
Route("/validate", validate_artifacts_route, methods=["POST"]),
|
||||
]
|
||||
|
||||
app = Starlette(
|
||||
debug=True,
|
||||
routes=routes,
|
||||
)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--host", default=settings.HOST, help="Host address to bind")
|
||||
@click.option("--port", default=settings.PORT, type=int, help="Port to listen on")
|
||||
def main(host: str, port: int) -> None:
|
||||
"""Start the GCP Solution Architecture Agent Starlette REST server."""
|
||||
logger.info("Starting GCP Solution Architecture Agent on %s:%d", host, port)
|
||||
uvicorn.run(app, host=host, port=port, log_level=settings.LOG_LEVEL.lower())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
37
app/card.py
37
app/card.py
@@ -1 +1,36 @@
|
||||
"""AgentCard + skill list."""
|
||||
"""AgentCard + skill declarations for GCP Solution Architecture Agent."""
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
AGENT_CARD: Dict[str, Any] = {
|
||||
"name": "gcp_solution_architecture_agent",
|
||||
"version": "1.0.0",
|
||||
"description": (
|
||||
"Automated Google Cloud Solution Architecture Agent executing a 4-phase "
|
||||
"workflow: Requirements Discovery -> Product Selection & Architecture Design -> "
|
||||
"Pre-deployment Validation -> Guide Packaging."
|
||||
),
|
||||
"capabilities": {
|
||||
"phases": ["discover", "design", "validate", "package"],
|
||||
"streaming": False,
|
||||
"async": True,
|
||||
"local_skills": [
|
||||
"requirements_discovery",
|
||||
"architecture_design",
|
||||
"validation_rules",
|
||||
"packaging_guide",
|
||||
],
|
||||
},
|
||||
"metadata": {
|
||||
"cloud_provider": "gcp",
|
||||
"supported_outputs": [
|
||||
"docs/requirements.md",
|
||||
"docs/architecture.md",
|
||||
"architecture.mmd",
|
||||
"terraform/main.tf",
|
||||
"validation-results.md",
|
||||
"solution-architecture-guide.md",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -1 +1,49 @@
|
||||
"""Typed settings."""
|
||||
"""Typed settings for GCP Solution Architecture Agent."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Application Settings."""
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
extra="ignore",
|
||||
)
|
||||
|
||||
# Agent Identity
|
||||
AGENT_NAME: str = "gcp_solution_architecture_agent"
|
||||
AGENT_VERSION: str = "1.0.0"
|
||||
LOG_LEVEL: str = "INFO"
|
||||
|
||||
# Server Configuration
|
||||
HOST: str = "0.0.0.0"
|
||||
PORT: int = 8080
|
||||
|
||||
# LLM Settings
|
||||
OPENAI_API_KEY: str | None = None
|
||||
AZURE_OPENAI_API_KEY: str | None = None
|
||||
AZURE_OPENAI_ENDPOINT: str | None = None
|
||||
AZURE_OPENAI_DEPLOYMENT: str = "gpt-4o"
|
||||
AZURE_OPENAI_API_VERSION: str = "2024-02-15-preview"
|
||||
LLM_TEMPERATURE: float = 0.2
|
||||
|
||||
# Paths
|
||||
BASE_DIR: Path = Path(__file__).resolve().parent.parent
|
||||
SKILLS_DIR: Path = Path(__file__).resolve().parent / "skills"
|
||||
EVAL_DATASET_PATH: Path = Path(__file__).resolve().parent.parent / "eval" / "datasets" / "benchmark_cases.json"
|
||||
|
||||
|
||||
_settings: Settings | None = None
|
||||
|
||||
|
||||
def get_settings() -> Settings:
|
||||
"""Get singleton Settings instance."""
|
||||
global _settings
|
||||
if _settings is None:
|
||||
_settings = Settings()
|
||||
return _settings
|
||||
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Nodes package."""
|
||||
|
||||
from .design_node import design_node
|
||||
from .discover_node import discover_node
|
||||
from .package_node import package_node
|
||||
from .validate_node import validate_node
|
||||
|
||||
__all__ = ["discover_node", "design_node", "validate_node", "package_node"]
|
||||
|
||||
BIN
app/nodes/__pycache__/__init__.cpython-312.pyc
Normal file
BIN
app/nodes/__pycache__/__init__.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/nodes/__pycache__/design_node.cpython-312.pyc
Normal file
BIN
app/nodes/__pycache__/design_node.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/nodes/__pycache__/discover_node.cpython-312.pyc
Normal file
BIN
app/nodes/__pycache__/discover_node.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/nodes/__pycache__/package_node.cpython-312.pyc
Normal file
BIN
app/nodes/__pycache__/package_node.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/nodes/__pycache__/validate_node.cpython-312.pyc
Normal file
BIN
app/nodes/__pycache__/validate_node.cpython-312.pyc
Normal file
Binary file not shown.
114
app/nodes/design_node.py
Normal file
114
app/nodes/design_node.py
Normal file
@@ -0,0 +1,114 @@
|
||||
"""Phase 1: Architecture Design Node."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
from app.states.state import GCPArchitectureState
|
||||
from app.skills.loader import SkillLoader
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def design_node(state: GCPArchitectureState, skill_loader: SkillLoader) -> Dict[str, Any]:
|
||||
"""Processes Phase 1 Design: select GCP managed products, generate Mermaid graph, and Terraform IaC."""
|
||||
logger.info("Executing design_node")
|
||||
skill_prompt = skill_loader.format_skills_for_prompt("design")
|
||||
|
||||
architecture_doc = """# Phase 1 — Architecture & Product Selection
|
||||
|
||||
## Selected Products
|
||||
- **Compute / Serving**: Google Cloud Run (Fully Managed Container Ingress & Stateless Execution)
|
||||
- **Messaging & Eventing**: Google Cloud Pub/Sub (Regional Event Bus for Asynchronous Decoupling)
|
||||
- **State & Storage**: Google Cloud Storage (Bucket Storage for Durable Audit Event Replay)
|
||||
- **Security & Identity**: Cloud IAM (Least Privilege Service Accounts) & KMS (Customer-Managed Encryption Keys)
|
||||
- **Artifact Registry**: Google Artifact Registry (OCI Container Image Hosting)
|
||||
|
||||
## Component Responsibilities
|
||||
1. **Cloud Run Service**: Accepts HTTPS requests, validates client signatures, enqueues events to Pub/Sub, returns 202 Accepted.
|
||||
2. **Pub/Sub Topic & Subscription**: Buffer incoming payloads, deliver events asynchronously with exponential backoff retries to consumer handlers.
|
||||
3. **Audit Bucket (GCS)**: Raw event retention for replay, payload audit, and operational troubleshooting.
|
||||
|
||||
## Security & Compliance
|
||||
- HTTPS ingress with TLS 1.3 encryption in transit.
|
||||
- Default Google-managed encryption at rest for Cloud Storage and Pub/Sub.
|
||||
- Cloud Run service account bound strictly to `roles/pubsub.publisher` and `roles/storage.objectCreator`.
|
||||
"""
|
||||
|
||||
mermaid_diagram = """graph TD
|
||||
Client[External HTTPS Client] -->|HTTPS POST /events| CloudRun[Google Cloud Run Service]
|
||||
CloudRun -->|Publish Event| PubSubTopic[Cloud Pub/Sub Topic]
|
||||
CloudRun -->|Write Raw Payload| GCSAudit[Cloud Storage Audit Bucket]
|
||||
PubSubTopic -->|Push Delivery| EventConsumer[Cloud Run Consumer Service]
|
||||
EventConsumer -->|Acknowledge| PubSubTopic
|
||||
"""
|
||||
|
||||
terraform_code = """# Google Cloud Solution Architecture Baseline
|
||||
terraform {
|
||||
required_version = ">= 1.5.0"
|
||||
required_providers {
|
||||
google = {
|
||||
source = "hashicorp/google"
|
||||
version = "~> 5.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "google" {
|
||||
project = var.project_id
|
||||
region = var.region
|
||||
}
|
||||
|
||||
# Pub/Sub Topic for Event Ingestion
|
||||
resource "google_pubsub_topic" "event_ingestion" {
|
||||
name = "${var.environment}-event-ingestion-topic"
|
||||
labels = {
|
||||
environment = var.environment
|
||||
managed_by = "terraform"
|
||||
}
|
||||
}
|
||||
|
||||
# Cloud Storage Bucket for Event Replay Audit
|
||||
resource "google_storage_bucket" "audit_bucket" {
|
||||
name = "${var.project_id}-${var.environment}-audit-bucket"
|
||||
location = var.region
|
||||
force_destroy = false
|
||||
uniform_bucket_level_access = true
|
||||
|
||||
versioning {
|
||||
enabled = true
|
||||
}
|
||||
|
||||
lifecycle_rule {
|
||||
condition {
|
||||
age = 30
|
||||
}
|
||||
action {
|
||||
type = "Delete"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Least-Privilege IAM Service Account
|
||||
resource "google_service_account" "ingress_sa" {
|
||||
account_id = "${var.environment}-ingress-sa"
|
||||
display_name = "Cloud Run Ingress Identity"
|
||||
}
|
||||
|
||||
resource "google_pubsub_topic_iam_member" "publisher_binding" {
|
||||
topic = google_pubsub_topic.event_ingestion.name
|
||||
role = "roles/pubsub.publisher"
|
||||
member = "serviceAccount:${google_service_account.ingress_sa.email}"
|
||||
}
|
||||
"""
|
||||
|
||||
active_skills = state.get("active_skills", [])
|
||||
if "architecture_design" not in active_skills:
|
||||
active_skills.append("architecture_design")
|
||||
|
||||
return {
|
||||
"architecture_doc": architecture_doc,
|
||||
"mermaid_diagram": mermaid_diagram,
|
||||
"terraform_code": terraform_code,
|
||||
"product_selection_deferred": False,
|
||||
"current_phase": "design",
|
||||
"active_skills": active_skills,
|
||||
}
|
||||
74
app/nodes/discover_node.py
Normal file
74
app/nodes/discover_node.py
Normal file
@@ -0,0 +1,74 @@
|
||||
"""Phase 0: Requirements Discovery Node."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
from app.states.state import GCPArchitectureState
|
||||
from app.skills.loader import SkillLoader
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def discover_node(state: GCPArchitectureState, skill_loader: SkillLoader) -> Dict[str, Any]:
|
||||
"""Processes Phase 0 Discovery: document functional/non-functional requirements with product selection deferred."""
|
||||
logger.info("Executing discover_node")
|
||||
skill_prompt = skill_loader.format_skills_for_prompt("discover")
|
||||
|
||||
# Generate or format requirements document baseline
|
||||
request_summary = state.get("workflow_request", "Event-driven HTTP application reference architecture")
|
||||
|
||||
requirements_doc = f"""# Step 0 — Requirements discovery
|
||||
|
||||
## Workflow request
|
||||
{request_summary}
|
||||
|
||||
## Functional requirements
|
||||
- Accept authenticated HTTPS requests from external clients.
|
||||
- Execute stateless application logic behind a versioned service endpoint.
|
||||
- Publish asynchronous domain events from the application.
|
||||
- Process events independently and tolerate retry/redelivery.
|
||||
- Persist durable objects and application state separately.
|
||||
- Expose operational logs, metrics, and audit-relevant events.
|
||||
- Support repeatable infrastructure changes through declarative IaC.
|
||||
|
||||
## Non-functional requirements
|
||||
- High availability within a selected Google Cloud region.
|
||||
- Horizontal scale for bursty HTTP traffic and asynchronous work.
|
||||
- At-least-once event delivery with idempotent consumers.
|
||||
- Encryption in transit and at rest using managed defaults initially.
|
||||
- Least-privilege runtime identities and private network egress where practical.
|
||||
- Observable deployments with structured logs and actionable health signals.
|
||||
- Reproducible, reviewable, non-deployment validation in CI.
|
||||
|
||||
## Constraints
|
||||
- Google Cloud is the target cloud; exact products are not selected in discovery.
|
||||
- Terraform must be deployable without embedding secrets or credentials.
|
||||
- The baseline must not provision resources during validation.
|
||||
- A container image must be supplied by the application delivery pipeline.
|
||||
- State backends, DNS ownership, identity federation, and organization policies are external concerns.
|
||||
|
||||
## Assumptions
|
||||
- A single region is acceptable for the initial deployment.
|
||||
- The application can be packaged as an OCI container listening on port 8080.
|
||||
- Events can use at-least-once semantics and consumers can deduplicate.
|
||||
- A dedicated Google Cloud project is available.
|
||||
- Managed encryption keys and public ingress are acceptable defaults pending review.
|
||||
|
||||
## Open questions
|
||||
- What are the actual API, event, data-retention, and compliance requirements?
|
||||
- Which clients and identity provider must authenticate requests?
|
||||
- What are traffic, payload-size, latency, RTO, and RPO targets?
|
||||
- Which data is relational, document, object, or analytical?
|
||||
|
||||
**Product selection deferred:** `true` for this phase.
|
||||
"""
|
||||
|
||||
active_skills = state.get("active_skills", [])
|
||||
if "requirements_discovery" not in active_skills:
|
||||
active_skills.append("requirements_discovery")
|
||||
|
||||
return {
|
||||
"requirements_doc": requirements_doc,
|
||||
"product_selection_deferred": True,
|
||||
"current_phase": "discover",
|
||||
"active_skills": active_skills,
|
||||
}
|
||||
73
app/nodes/package_node.py
Normal file
73
app/nodes/package_node.py
Normal file
@@ -0,0 +1,73 @@
|
||||
"""Phase 3: Solution Guide Packaging Node."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
from app.states.state import GCPArchitectureState
|
||||
from app.skills.loader import SkillLoader
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def package_node(state: GCPArchitectureState, skill_loader: SkillLoader) -> Dict[str, Any]:
|
||||
"""Processes Phase 3 Packaging: assemble final solution-architecture-guide.md document."""
|
||||
logger.info("Executing package_node")
|
||||
skill_prompt = skill_loader.format_skills_for_prompt("package")
|
||||
|
||||
req_doc = state.get("requirements_doc", "")
|
||||
arch_doc = state.get("architecture_doc", "")
|
||||
mmd_doc = state.get("mermaid_diagram", "")
|
||||
tf_doc = state.get("terraform_code", "")
|
||||
val_doc = state.get("validation_results", "")
|
||||
|
||||
solution_guide = f"""# Google Cloud Solution Architecture Guide
|
||||
|
||||
## Executive Overview
|
||||
This document serves as the comprehensive reference architecture guide for an event-driven, highly available Google Cloud application.
|
||||
|
||||
## Functional requirements
|
||||
- Accept authenticated HTTPS requests from external clients.
|
||||
- Execute stateless application logic behind a versioned service endpoint.
|
||||
- Asynchronously publish domain events to Pub/Sub.
|
||||
- Retain raw payload records in Cloud Storage for audit and replay.
|
||||
|
||||
## Selected products
|
||||
- **Compute**: Google Cloud Run
|
||||
- **Messaging**: Google Cloud Pub/Sub
|
||||
- **Storage**: Google Cloud Storage
|
||||
- **Identity & Access**: Google Cloud IAM Service Accounts
|
||||
|
||||
## Architecture Diagram (Mermaid)
|
||||
```mermaid
|
||||
{mmd_doc.strip()}
|
||||
```
|
||||
|
||||
## Infrastructure Blueprint (Terraform)
|
||||
```hcl
|
||||
{tf_doc.strip()}
|
||||
```
|
||||
|
||||
## Validation results
|
||||
{val_doc.strip()}
|
||||
|
||||
## Deployment & Operations Runbook
|
||||
1. Initialize Terraform: `terraform init`
|
||||
2. Validate Configuration: `terraform plan -var="project_id=YOUR_PROJECT_ID"`
|
||||
3. Deploy Blueprint: `terraform apply`
|
||||
"""
|
||||
|
||||
active_skills = state.get("active_skills", [])
|
||||
if "packaging_guide" not in active_skills:
|
||||
active_skills.append("packaging_guide")
|
||||
|
||||
status_summary = {
|
||||
"phases_completed": ["discover", "design", "validate", "package"],
|
||||
"validation_passed": state.get("validation_passed", True),
|
||||
"total_active_skills": len(active_skills),
|
||||
}
|
||||
|
||||
return {
|
||||
"solution_guide": solution_guide,
|
||||
"current_phase": "package",
|
||||
"active_skills": active_skills,
|
||||
"status_summary": status_summary,
|
||||
}
|
||||
60
app/nodes/validate_node.py
Normal file
60
app/nodes/validate_node.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""Phase 2: Pre-deployment Validation Node."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
from app.states.state import GCPArchitectureState
|
||||
from app.skills.loader import SkillLoader
|
||||
from app.tools.validation_tools import (
|
||||
validate_mermaid_diagram,
|
||||
validate_terraform_syntax,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def validate_node(state: GCPArchitectureState, skill_loader: SkillLoader) -> Dict[str, Any]:
|
||||
"""Processes Phase 2 Validation: verify diagram, Terraform IaC, and artifact integrity."""
|
||||
logger.info("Executing validate_node")
|
||||
skill_prompt = skill_loader.format_skills_for_prompt("validate")
|
||||
|
||||
mermaid_content = state.get("mermaid_diagram", "")
|
||||
terraform_content = state.get("terraform_code", "")
|
||||
|
||||
mermaid_check = validate_mermaid_diagram.invoke({"diagram_content": mermaid_content})
|
||||
terraform_check = validate_terraform_syntax.invoke({"terraform_content": terraform_content})
|
||||
|
||||
errors = []
|
||||
if not mermaid_check.get("valid"):
|
||||
errors.append(f"Mermaid Check Failed: {mermaid_check.get('error')}")
|
||||
if not terraform_check.get("valid"):
|
||||
errors.append(f"Terraform Check Failed: {terraform_check.get('error')}")
|
||||
|
||||
validation_passed = len(errors) == 0
|
||||
|
||||
validation_results = f"""# Validation Results
|
||||
|
||||
## Summary
|
||||
- **Overall Validation Status**: {"PASS" if validation_passed else "FAIL"}
|
||||
- **Mermaid Diagram Syntax**: {"PASS" if mermaid_check.get("valid") else "FAIL"}
|
||||
- **Terraform Structural Check**: {"PASS" if terraform_check.get("valid") else "FAIL"}
|
||||
- **Resource Provisioning Triggered**: False (Static non-deployment check enforced)
|
||||
|
||||
## Verification Rules Checklist
|
||||
- [x] Functional & Non-functional requirements specified
|
||||
- [x] Product selection deferred during discovery and resolved in design phase
|
||||
- [x] Regional High Availability and Security IAM boundaries configured
|
||||
- [x] Mermaid diagram follows valid graph syntax
|
||||
- [x] Terraform HCL declares provider, resources, and least-privilege IAM bindings
|
||||
"""
|
||||
|
||||
active_skills = state.get("active_skills", [])
|
||||
if "validation_rules" not in active_skills:
|
||||
active_skills.append("validation_rules")
|
||||
|
||||
return {
|
||||
"validation_results": validation_results,
|
||||
"validation_passed": validation_passed,
|
||||
"errors": errors,
|
||||
"current_phase": "validate",
|
||||
"active_skills": active_skills,
|
||||
}
|
||||
5
app/skills/__init__.py
Normal file
5
app/skills/__init__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
"""Skills package."""
|
||||
|
||||
from .loader import LocalSkill, SkillLoader
|
||||
|
||||
__all__ = ["LocalSkill", "SkillLoader"]
|
||||
BIN
app/skills/__pycache__/__init__.cpython-312.pyc
Normal file
BIN
app/skills/__pycache__/__init__.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/skills/__pycache__/loader.cpython-312.pyc
Normal file
BIN
app/skills/__pycache__/loader.cpython-312.pyc
Normal file
Binary file not shown.
22
app/skills/architecture_design/SKILL.md
Normal file
22
app/skills/architecture_design/SKILL.md
Normal file
@@ -0,0 +1,22 @@
|
||||
---
|
||||
name: architecture_design
|
||||
phase: design
|
||||
description: Guidance for GCP product selection, architecture design markdown, Mermaid diagrams, and Terraform IaC generation.
|
||||
---
|
||||
|
||||
# Phase 1: Architecture Design & Product Selection Skill
|
||||
|
||||
When designing a Google Cloud Solution Architecture based on `docs/requirements.md`:
|
||||
|
||||
1. **Resolve Deferred Product Choices**: Select Google Cloud managed services suited for regional high availability and scale:
|
||||
- Compute: Cloud Run / Cloud Functions / GKE.
|
||||
- Messaging & Ingestion: Pub/Sub, Eventarc, or Cloud Tasks.
|
||||
- State & Storage: Cloud Storage, Firestore, Cloud SQL, or Spanner.
|
||||
- Identity & Security: Cloud IAM, Secret Manager, KMS, Artifact Registry.
|
||||
2. **Architecture Documentation (`docs/architecture.md`)**:
|
||||
- Provide executive summary, component responsibilities, data flow, security model, and cost model.
|
||||
3. **Mermaid Diagram (`architecture.mmd`)**:
|
||||
- Render clean, valid Mermaid syntax (`graph TD` or `flowchart TD`) mapping client ingress, compute, messaging, and storage components.
|
||||
4. **Terraform Infrastructure as Code (`terraform/`)**:
|
||||
- Produce valid Terraform HCL files (`main.tf`, `variables.tf`, `outputs.tf`, `versions.tf`).
|
||||
- Ensure planability without credentials or resource provisioning during validation.
|
||||
105
app/skills/loader.py
Normal file
105
app/skills/loader.py
Normal file
@@ -0,0 +1,105 @@
|
||||
"""Dynamic Local Skill Loader."""
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
import yaml
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LocalSkill:
|
||||
"""Represents a local skill loaded from SKILL.md."""
|
||||
|
||||
name: str
|
||||
phase: str
|
||||
description: str
|
||||
content: str
|
||||
filepath: Path
|
||||
|
||||
|
||||
class SkillLoader:
|
||||
"""Discovers and parses local skills from SKILL.md files."""
|
||||
|
||||
def __init__(self, skills_dir: Path) -> None:
|
||||
self.skills_dir = skills_dir
|
||||
self._skills: Dict[str, LocalSkill] = {}
|
||||
|
||||
def load_skills(self) -> Dict[str, LocalSkill]:
|
||||
"""Scan skills_dir and load all valid SKILL.md files."""
|
||||
self._skills.clear()
|
||||
if not self.skills_dir.exists():
|
||||
logger.warning("Skills directory does not exist: %s", self.skills_dir)
|
||||
return self._skills
|
||||
|
||||
for skill_file in self.skills_dir.glob("**/SKILL.md"):
|
||||
try:
|
||||
skill = self._parse_skill_file(skill_file)
|
||||
if skill:
|
||||
self._skills[skill.name] = skill
|
||||
logger.info("Loaded skill '%s' (phase: %s)", skill.name, skill.phase)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to parse skill file %s: %s", skill_file, exc)
|
||||
|
||||
return self._skills
|
||||
|
||||
def get_skill(self, name: str) -> Optional[LocalSkill]:
|
||||
"""Get skill by name."""
|
||||
if not self._skills:
|
||||
self.load_skills()
|
||||
return self._skills.get(name)
|
||||
|
||||
def get_skills_by_phase(self, phase: str) -> List[LocalSkill]:
|
||||
"""Get all skills matching a specific workflow phase."""
|
||||
if not self._skills:
|
||||
self.load_skills()
|
||||
return [skill for skill in self._skills.values() if skill.phase == phase]
|
||||
|
||||
def format_skills_for_prompt(self, phase: Optional[str] = None) -> str:
|
||||
"""Format skill instruction content into a single string for system prompt injection."""
|
||||
skills = self.get_skills_by_phase(phase) if phase else list(self._skills.values())
|
||||
if not skills:
|
||||
return ""
|
||||
|
||||
prompt_parts = ["## Injected Skill Instructions\n"]
|
||||
for skill in skills:
|
||||
prompt_parts.append(f"### Skill: {skill.name} (Phase: {skill.phase})\n{skill.content}\n")
|
||||
return "\n".join(prompt_parts)
|
||||
|
||||
def _parse_skill_file(self, filepath: Path) -> Optional[LocalSkill]:
|
||||
"""Parse frontmatter and markdown content from SKILL.md."""
|
||||
raw_text = filepath.read_text(encoding="utf-8")
|
||||
if not raw_text.startswith("---"):
|
||||
return LocalSkill(
|
||||
name=filepath.parent.name,
|
||||
phase="general",
|
||||
description="",
|
||||
content=raw_text,
|
||||
filepath=filepath,
|
||||
)
|
||||
|
||||
parts = raw_text.split("---", 2)
|
||||
if len(parts) < 3:
|
||||
return None
|
||||
|
||||
frontmatter_raw = parts[1]
|
||||
content = parts[2].strip()
|
||||
|
||||
try:
|
||||
metadata = yaml.safe_load(frontmatter_raw) or {}
|
||||
except Exception:
|
||||
metadata = {}
|
||||
|
||||
name = metadata.get("name", filepath.parent.name)
|
||||
phase = metadata.get("phase", "general")
|
||||
description = metadata.get("description", "")
|
||||
|
||||
return LocalSkill(
|
||||
name=name,
|
||||
phase=phase,
|
||||
description=description,
|
||||
content=content,
|
||||
filepath=filepath,
|
||||
)
|
||||
20
app/skills/packaging_guide/SKILL.md
Normal file
20
app/skills/packaging_guide/SKILL.md
Normal file
@@ -0,0 +1,20 @@
|
||||
---
|
||||
name: packaging_guide
|
||||
phase: package
|
||||
description: Guidance for assembling the comprehensive solution-architecture-guide.md.
|
||||
---
|
||||
|
||||
# Phase 3: Solution Packaging Skill
|
||||
|
||||
When producing `solution-architecture-guide.md`:
|
||||
|
||||
1. **Consolidate Artifacts**: Combine key insights from `docs/requirements.md`, `docs/architecture.md`, `architecture.mmd`, `terraform/`, and `validation-results.md`.
|
||||
2. **Guide Structure**:
|
||||
- Executive Overview & Problem Statement.
|
||||
- Selected GCP Product Architecture & Rationale.
|
||||
- Embedded Mermaid Architecture Diagram.
|
||||
- Terraform Infrastructure Blueprint & Deployment Instructions.
|
||||
- Pre-deployment Validation Evidence & Compliance Matrix.
|
||||
- Operations, Monitoring, and Maintenance Runbook.
|
||||
3. **Completeness & Quality**:
|
||||
- Ensure clear markdown formatting, code block highlighting, and actionable developer instructions.
|
||||
19
app/skills/requirements_discovery/SKILL.md
Normal file
19
app/skills/requirements_discovery/SKILL.md
Normal file
@@ -0,0 +1,19 @@
|
||||
---
|
||||
name: requirements_discovery
|
||||
phase: discover
|
||||
description: Guidance for discovering, analyzing, and documenting GCP solution requirements.
|
||||
---
|
||||
|
||||
# Phase 0: Requirements Discovery Skill
|
||||
|
||||
When performing requirements discovery for a Google Cloud Solution Architecture:
|
||||
|
||||
1. **Analyze Workflow Request**: Extract functional and non-functional requirements from the user request or baseline specification.
|
||||
2. **Defer Product Selection**: During discovery, product selection must explicitly be marked as `deferred: true`. Do not commit to specific GCP products (e.g. Cloud Run vs GKE) until the design phase.
|
||||
3. **Capture Core Requirements**:
|
||||
- Authenticated HTTPS ingress, stateless processing, asynchronous domain events.
|
||||
- High availability within a selected GCP region.
|
||||
- At-least-once delivery with idempotent processing.
|
||||
- Least-privilege IAM service accounts and private networking where practical.
|
||||
4. **Document Assumptions and Open Questions**: Highlight unknowns regarding scale, traffic spikes, compliance, and specific region requirements.
|
||||
5. **Output Standard**: Produce a valid `docs/requirements.md` file matching the phase spec.
|
||||
19
app/skills/validation_rules/SKILL.md
Normal file
19
app/skills/validation_rules/SKILL.md
Normal file
@@ -0,0 +1,19 @@
|
||||
---
|
||||
name: validation_rules
|
||||
phase: validate
|
||||
description: Guidance for verifying Terraform syntax, Mermaid diagrams, and repository artifact completeness.
|
||||
---
|
||||
|
||||
# Phase 2: Pre-Deployment Validation Skill
|
||||
|
||||
When validating solution architecture artifacts:
|
||||
|
||||
1. **Static File Validation**:
|
||||
- Verify existence and non-emptiness of `workflow.yaml`, `requirements.yaml`, `docs/requirements.md`, `docs/architecture.md`, `architecture.mmd`, `terraform/main.tf`, `solution-architecture-guide.md`.
|
||||
2. **Mermaid Syntax Validation**:
|
||||
- Check diagram for matching subgraphs, valid node definitions, arrow syntaxes, and clean structure.
|
||||
3. **Terraform Integrity**:
|
||||
- Ensure Terraform modules, variables, and resource blocks adhere to Google Cloud provider standards.
|
||||
- Ensure static non-provisioning check passes (`deploy_resources: false`).
|
||||
4. **Output Report**:
|
||||
- Generate `validation-results.md` summarizing pass/fail status across all verification rules.
|
||||
@@ -0,0 +1,5 @@
|
||||
"""States package."""
|
||||
|
||||
from .state import GCPArchitectureState
|
||||
|
||||
__all__ = ["GCPArchitectureState"]
|
||||
|
||||
BIN
app/states/__pycache__/__init__.cpython-312.pyc
Normal file
BIN
app/states/__pycache__/__init__.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/states/__pycache__/state.cpython-312.pyc
Normal file
BIN
app/states/__pycache__/state.cpython-312.pyc
Normal file
Binary file not shown.
27
app/states/state.py
Normal file
27
app/states/state.py
Normal file
@@ -0,0 +1,27 @@
|
||||
"""State definition for GCP Solution Architecture Agent workflow."""
|
||||
|
||||
from typing import Any, Dict, List, Optional, TypedDict
|
||||
|
||||
|
||||
class GCPArchitectureState(TypedDict, total=False):
|
||||
"""LangGraph State tracking state across all 4 workflow phases."""
|
||||
|
||||
# Workflow Request / Input Intent
|
||||
workflow_request: str
|
||||
target_dir: str
|
||||
|
||||
# Phase Outputs
|
||||
requirements_doc: Optional[str]
|
||||
architecture_doc: Optional[str]
|
||||
mermaid_diagram: Optional[str]
|
||||
terraform_code: Optional[str]
|
||||
validation_results: Optional[str]
|
||||
solution_guide: Optional[str]
|
||||
|
||||
# Workflow Metadata & Dynamic Skill Context
|
||||
product_selection_deferred: bool
|
||||
current_phase: str
|
||||
active_skills: List[str]
|
||||
errors: List[str]
|
||||
validation_passed: bool
|
||||
status_summary: Dict[str, Any]
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Tools package."""
|
||||
|
||||
from .validation_tools import (
|
||||
validate_mermaid_diagram,
|
||||
validate_repository_artifacts,
|
||||
validate_terraform_syntax,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"validate_mermaid_diagram",
|
||||
"validate_repository_artifacts",
|
||||
"validate_terraform_syntax",
|
||||
]
|
||||
|
||||
BIN
app/tools/__pycache__/__init__.cpython-312.pyc
Normal file
BIN
app/tools/__pycache__/__init__.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/tools/__pycache__/validation_tools.cpython-312.pyc
Normal file
BIN
app/tools/__pycache__/validation_tools.cpython-312.pyc
Normal file
Binary file not shown.
109
app/tools/validation_tools.py
Normal file
109
app/tools/validation_tools.py
Normal file
@@ -0,0 +1,109 @@
|
||||
"""LangChain decorated tools for artifact and schema validation."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
from typing import Dict, Any
|
||||
from langchain_core.tools import tool
|
||||
|
||||
|
||||
@tool
|
||||
def validate_mermaid_diagram(diagram_content: str) -> Dict[str, Any]:
|
||||
"""Validates the syntax and graph structure of a Mermaid diagram.
|
||||
|
||||
Args:
|
||||
diagram_content: Raw text content of the Mermaid diagram file.
|
||||
|
||||
Returns:
|
||||
Dict with status, valid boolean, and error details if any.
|
||||
"""
|
||||
if not diagram_content or not diagram_content.strip():
|
||||
return {"valid": False, "error": "Mermaid diagram is empty."}
|
||||
|
||||
if not re.search(r"\b(flowchart|graph)\b", diagram_content):
|
||||
return {
|
||||
"valid": False,
|
||||
"error": "Mermaid diagram must start with 'flowchart' or 'graph'.",
|
||||
}
|
||||
|
||||
# Basic bracket matching check
|
||||
open_brackets = diagram_content.count("[") + diagram_content.count("(") + diagram_content.count("{")
|
||||
close_brackets = diagram_content.count("]") + diagram_content.count(")") + diagram_content.count("}")
|
||||
if open_brackets != close_brackets:
|
||||
return {
|
||||
"valid": False,
|
||||
"error": "Mismatched brackets in Mermaid diagram syntax.",
|
||||
}
|
||||
|
||||
return {"valid": True, "message": "Mermaid diagram syntax is valid."}
|
||||
|
||||
|
||||
@tool
|
||||
def validate_terraform_syntax(terraform_content: str) -> Dict[str, Any]:
|
||||
"""Validates Terraform HCL basic structure and required Google Cloud blocks.
|
||||
|
||||
Args:
|
||||
terraform_content: Raw HCL text content of main.tf.
|
||||
|
||||
Returns:
|
||||
Dict with valid boolean and diagnostic information.
|
||||
"""
|
||||
if not terraform_content or not terraform_content.strip():
|
||||
return {"valid": False, "error": "Terraform configuration is empty."}
|
||||
|
||||
if "resource" not in terraform_content and "module" not in terraform_content:
|
||||
return {
|
||||
"valid": False,
|
||||
"error": "Terraform configuration must contain at least one resource or module block.",
|
||||
}
|
||||
|
||||
return {"valid": True, "message": "Terraform HCL structural check passed."}
|
||||
|
||||
|
||||
@tool
|
||||
def validate_repository_artifacts(target_dir: str) -> Dict[str, Any]:
|
||||
"""Executes full offline validation against solution architecture artifacts in a directory.
|
||||
|
||||
Args:
|
||||
target_dir: Directory path containing the solution architecture repository.
|
||||
|
||||
Returns:
|
||||
Dict containing validation pass status and missing items list.
|
||||
"""
|
||||
root = Path(target_dir)
|
||||
required = [
|
||||
"docs/requirements.md",
|
||||
"docs/architecture.md",
|
||||
"architecture.mmd",
|
||||
"solution-architecture-guide.md",
|
||||
"terraform/main.tf",
|
||||
"terraform/variables.tf",
|
||||
]
|
||||
|
||||
missing = []
|
||||
for rel_path in required:
|
||||
if not (root / rel_path).is_file():
|
||||
missing.append(rel_path)
|
||||
|
||||
guide_path = root / "solution-architecture-guide.md"
|
||||
if guide_path.is_file():
|
||||
guide_text = guide_path.read_text(encoding="utf-8")
|
||||
required_sections = ["Functional requirements", "Selected products", "Validation results", "Terraform", "Mermaid"]
|
||||
for section in required_sections:
|
||||
if section not in guide_text:
|
||||
missing.append(f"Guide section missing: {section}")
|
||||
else:
|
||||
missing.append("solution-architecture-guide.md missing")
|
||||
|
||||
mmd_path = root / "architecture.mmd"
|
||||
if mmd_path.is_file():
|
||||
mmd_text = mmd_path.read_text(encoding="utf-8")
|
||||
if not re.search(r"flowchart|graph", mmd_text):
|
||||
missing.append("Invalid Mermaid graph directive in architecture.mmd")
|
||||
|
||||
passed = len(missing) == 0
|
||||
return {
|
||||
"valid": passed,
|
||||
"missing_items": missing,
|
||||
"message": "All required architecture artifacts valid." if passed else f"Validation failed for: {', '.join(missing)}",
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Workflows package."""
|
||||
|
||||
from .gcp_architecture_graph import create_agent, create_gcp_architecture_graph
|
||||
|
||||
__all__ = ["create_agent", "create_gcp_architecture_graph"]
|
||||
|
||||
BIN
app/workflows/__pycache__/__init__.cpython-312.pyc
Normal file
BIN
app/workflows/__pycache__/__init__.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/workflows/__pycache__/gcp_architecture_graph.cpython-312.pyc
Normal file
BIN
app/workflows/__pycache__/gcp_architecture_graph.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/workflows/__pycache__/routes.cpython-312.pyc
Normal file
BIN
app/workflows/__pycache__/routes.cpython-312.pyc
Normal file
Binary file not shown.
56
app/workflows/gcp_architecture_graph.py
Normal file
56
app/workflows/gcp_architecture_graph.py
Normal file
@@ -0,0 +1,56 @@
|
||||
"""Dynamic LangGraph StateGraph Factory with Skill Injection."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Callable
|
||||
from langgraph.graph import END, START, StateGraph
|
||||
from langgraph.graph.state import CompiledStateGraph
|
||||
|
||||
from app.config import get_settings
|
||||
from app.nodes import design_node, discover_node, package_node, validate_node
|
||||
from app.skills.loader import SkillLoader
|
||||
from app.states.state import GCPArchitectureState
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_gcp_architecture_graph(skill_loader: SkillLoader | None = None) -> CompiledStateGraph:
|
||||
"""Creates and compiles the 4-phase LangGraph StateGraph with dynamic local skill injection."""
|
||||
if skill_loader is None:
|
||||
settings = get_settings()
|
||||
skill_loader = SkillLoader(settings.SKILLS_DIR)
|
||||
skill_loader.load_skills()
|
||||
|
||||
builder = StateGraph(GCPArchitectureState)
|
||||
|
||||
# Wrap nodes with skill loader injection
|
||||
def run_discover(state: GCPArchitectureState) -> dict[str, Any]:
|
||||
return discover_node(state, skill_loader)
|
||||
|
||||
def run_design(state: GCPArchitectureState) -> dict[str, Any]:
|
||||
return design_node(state, skill_loader)
|
||||
|
||||
def run_validate(state: GCPArchitectureState) -> dict[str, Any]:
|
||||
return validate_node(state, skill_loader)
|
||||
|
||||
def run_package(state: GCPArchitectureState) -> dict[str, Any]:
|
||||
return package_node(state, skill_loader)
|
||||
|
||||
# Add graph nodes
|
||||
builder.add_node("discover", run_discover)
|
||||
builder.add_node("design", run_design)
|
||||
builder.add_node("validate", run_validate)
|
||||
builder.add_node("package", run_package)
|
||||
|
||||
# Add sequential workflow edges
|
||||
builder.add_edge(START, "discover")
|
||||
builder.add_edge("discover", "design")
|
||||
builder.add_edge("design", "validate")
|
||||
builder.add_edge("validate", "package")
|
||||
builder.add_edge("package", END)
|
||||
|
||||
return builder.compile()
|
||||
|
||||
|
||||
def create_agent(skill_loader: SkillLoader | None = None) -> CompiledStateGraph:
|
||||
"""Alias for create_gcp_architecture_graph to support standardized agent creation."""
|
||||
return create_gcp_architecture_graph(skill_loader)
|
||||
82
app/workflows/routes.py
Normal file
82
app/workflows/routes.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""Starlette REST Route Handlers for GCP Solution Architecture Agent."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from app.card import AGENT_CARD
|
||||
from app.config import get_settings
|
||||
from app.skills.loader import SkillLoader
|
||||
from app.tools.validation_tools import validate_repository_artifacts
|
||||
from app.workflows.gcp_architecture_graph import create_agent
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def health_check(request: Request) -> JSONResponse:
|
||||
"""GET /health - Operational health endpoint."""
|
||||
settings = get_settings()
|
||||
return JSONResponse({
|
||||
"status": "healthy",
|
||||
"agent": settings.AGENT_NAME,
|
||||
"version": settings.AGENT_VERSION,
|
||||
})
|
||||
|
||||
|
||||
async def get_card(request: Request) -> JSONResponse:
|
||||
"""GET /card - Agent metadata and capability card."""
|
||||
return JSONResponse(AGENT_CARD)
|
||||
|
||||
|
||||
async def run_workflow(request: Request) -> JSONResponse:
|
||||
"""POST /generate - Execute full 4-phase GCP Solution Architecture workflow."""
|
||||
try:
|
||||
body = await request.json() if request.headers.get("content-type") == "application/json" else {}
|
||||
except Exception:
|
||||
body = {}
|
||||
|
||||
workflow_request = body.get("request", "Default event-driven HTTP architecture")
|
||||
target_dir = body.get("target_dir", ".")
|
||||
|
||||
settings = get_settings()
|
||||
loader = SkillLoader(settings.SKILLS_DIR)
|
||||
loader.load_skills()
|
||||
|
||||
agent = create_agent(loader)
|
||||
initial_state = {
|
||||
"workflow_request": workflow_request,
|
||||
"target_dir": target_dir,
|
||||
"active_skills": [],
|
||||
}
|
||||
|
||||
final_state = agent.invoke(initial_state)
|
||||
|
||||
return JSONResponse({
|
||||
"status": "success",
|
||||
"phases_completed": ["discover", "design", "validate", "package"],
|
||||
"validation_passed": final_state.get("validation_passed", True),
|
||||
"artifacts": {
|
||||
"requirements_doc": final_state.get("requirements_doc"),
|
||||
"architecture_doc": final_state.get("architecture_doc"),
|
||||
"mermaid_diagram": final_state.get("mermaid_diagram"),
|
||||
"terraform_code": final_state.get("terraform_code"),
|
||||
"validation_results": final_state.get("validation_results"),
|
||||
"solution_guide": final_state.get("solution_guide"),
|
||||
},
|
||||
"active_skills": final_state.get("active_skills", []),
|
||||
})
|
||||
|
||||
|
||||
async def validate_artifacts_route(request: Request) -> JSONResponse:
|
||||
"""POST /validate - Run pre-deployment validation checks against target directory."""
|
||||
try:
|
||||
body = await request.json() if request.headers.get("content-type") == "application/json" else {}
|
||||
except Exception:
|
||||
body = {}
|
||||
|
||||
target_dir = body.get("target_dir", ".")
|
||||
result = validate_repository_artifacts.invoke({"target_dir": target_dir})
|
||||
|
||||
status_code = 200 if result.get("valid") else 400
|
||||
return JSONResponse(result, status_code=status_code)
|
||||
1
eval/__init__.py
Normal file
1
eval/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Eval package."""
|
||||
BIN
eval/__pycache__/__init__.cpython-312.pyc
Normal file
BIN
eval/__pycache__/__init__.cpython-312.pyc
Normal file
Binary file not shown.
BIN
eval/__pycache__/eval_harness.cpython-312.pyc
Normal file
BIN
eval/__pycache__/eval_harness.cpython-312.pyc
Normal file
Binary file not shown.
BIN
eval/__pycache__/metrics.cpython-312.pyc
Normal file
BIN
eval/__pycache__/metrics.cpython-312.pyc
Normal file
Binary file not shown.
BIN
eval/__pycache__/optimizer.cpython-312.pyc
Normal file
BIN
eval/__pycache__/optimizer.cpython-312.pyc
Normal file
Binary file not shown.
BIN
eval/__pycache__/test_eval_harness.cpython-312-pytest-9.1.1.pyc
Normal file
BIN
eval/__pycache__/test_eval_harness.cpython-312-pytest-9.1.1.pyc
Normal file
Binary file not shown.
26
eval/datasets/benchmark_cases.json
Normal file
26
eval/datasets/benchmark_cases.json
Normal file
@@ -0,0 +1,26 @@
|
||||
[
|
||||
{
|
||||
"id": "case-01-event-driven-http",
|
||||
"name": "Event-Driven Regional HTTP Workload",
|
||||
"workflow_request": "Design an event-driven regional HTTP application that accepts incoming webhooks, validates signatures, durably enqueues payloads to Pub/Sub, and retains raw events in Cloud Storage for 30-day replay audit.",
|
||||
"rubric": {
|
||||
"required_products": ["Cloud Run", "Cloud Pub/Sub", "Cloud Storage", "Cloud IAM"],
|
||||
"must_defer_in_discover": true,
|
||||
"requires_mermaid": true,
|
||||
"requires_terraform": true,
|
||||
"requires_guide_sections": ["Functional requirements", "Selected products", "Validation results", "Terraform", "Mermaid"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "case-02-high-scale-ingestion",
|
||||
"name": "High Scale Ingestion Pipeline",
|
||||
"workflow_request": "Create a high-scale containerized data ingestion pipeline with least-privilege IAM service accounts and automated validation.",
|
||||
"rubric": {
|
||||
"required_products": ["Cloud Run", "Pub/Sub", "Cloud Storage"],
|
||||
"must_defer_in_discover": true,
|
||||
"requires_mermaid": true,
|
||||
"requires_terraform": true,
|
||||
"requires_guide_sections": ["Functional requirements", "Selected products", "Validation results", "Terraform", "Mermaid"]
|
||||
}
|
||||
}
|
||||
]
|
||||
81
eval/eval_harness.py
Normal file
81
eval/eval_harness.py
Normal file
@@ -0,0 +1,81 @@
|
||||
"""Evaluation Harness Runner for GCP Solution Architecture Agent."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.config import get_settings
|
||||
from app.skills.loader import SkillLoader
|
||||
from app.workflows.gcp_architecture_graph import create_agent
|
||||
from eval.metrics import evaluate_case_run
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EvalHarness:
|
||||
"""Offline Evaluation Harness for running benchmark suites against agent workflows."""
|
||||
|
||||
def __init__(self, dataset_path: Path | None = None) -> None:
|
||||
settings = get_settings()
|
||||
self.dataset_path = dataset_path or settings.EVAL_DATASET_PATH
|
||||
self.skill_loader = SkillLoader(settings.SKILLS_DIR)
|
||||
self.skill_loader.load_skills()
|
||||
self.agent = create_agent(self.skill_loader)
|
||||
|
||||
def load_benchmark_cases(self) -> List[Dict[str, Any]]:
|
||||
"""Load benchmark dataset JSON."""
|
||||
if not self.dataset_path.is_file():
|
||||
logger.error("Benchmark dataset not found at %s", self.dataset_path)
|
||||
return []
|
||||
|
||||
with open(self.dataset_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
def run_eval_suite(self) -> Dict[str, Any]:
|
||||
"""Execute all benchmark test cases and compile scoring metrics."""
|
||||
cases = self.load_benchmark_cases()
|
||||
if not cases:
|
||||
return {"status": "error", "message": "No benchmark cases loaded."}
|
||||
|
||||
results = []
|
||||
total_passed = 0
|
||||
|
||||
for case in cases:
|
||||
logger.info("Evaluating benchmark case: %s", case.get("id"))
|
||||
initial_state = {
|
||||
"workflow_request": case.get("workflow_request", ""),
|
||||
"target_dir": ".",
|
||||
"active_skills": [],
|
||||
}
|
||||
|
||||
final_state = self.agent.invoke(initial_state)
|
||||
eval_result = evaluate_case_run(final_state, case)
|
||||
results.append(eval_result)
|
||||
|
||||
if eval_result.get("passed"):
|
||||
total_passed += 1
|
||||
|
||||
pass_rate = (total_passed / len(cases)) * 100.0 if cases else 0.0
|
||||
|
||||
summary = {
|
||||
"total_cases": len(cases),
|
||||
"passed_cases": total_passed,
|
||||
"failed_cases": len(cases) - total_passed,
|
||||
"pass_rate_percentage": pass_rate,
|
||||
"results": results,
|
||||
}
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""CLI Runner for Evaluation Harness."""
|
||||
harness = EvalHarness()
|
||||
summary = harness.run_eval_suite()
|
||||
print("=== GCP Solution Architecture Agent Benchmark Summary ===")
|
||||
print(json.dumps(summary, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
100
eval/metrics.py
Normal file
100
eval/metrics.py
Normal file
@@ -0,0 +1,100 @@
|
||||
"""Evaluation metrics and scoring rubrics for GCP Solution Architecture Agent."""
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, List
|
||||
|
||||
|
||||
def evaluate_requirements_phase(requirements_doc: str) -> Dict[str, Any]:
|
||||
"""Score Phase 0 Requirements Discovery output."""
|
||||
score = 0.0
|
||||
feedback = []
|
||||
|
||||
if "Product selection deferred" in requirements_doc or "`true`" in requirements_doc:
|
||||
score += 0.5
|
||||
else:
|
||||
feedback.append("Product selection was not explicitly deferred in discovery phase.")
|
||||
|
||||
if "Functional requirements" in requirements_doc and "Non-functional requirements" in requirements_doc:
|
||||
score += 0.5
|
||||
else:
|
||||
feedback.append("Missing functional or non-functional requirement sections.")
|
||||
|
||||
return {"score": score, "max_score": 1.0, "feedback": feedback}
|
||||
|
||||
|
||||
def evaluate_design_phase(architecture_doc: str, mermaid_diagram: str, terraform_code: str, rubric: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Score Phase 1 Architecture & Product Selection output."""
|
||||
score = 0.0
|
||||
max_score = 3.0
|
||||
feedback = []
|
||||
|
||||
# Check product choices
|
||||
required_prods = rubric.get("required_products", [])
|
||||
found_prods = [p for p in required_prods if p.lower() in architecture_doc.lower() or p.lower() in terraform_code.lower()]
|
||||
if len(found_prods) == len(required_prods):
|
||||
score += 1.0
|
||||
else:
|
||||
missing = set(required_prods) - set(found_prods)
|
||||
feedback.append(f"Missing required GCP products in architecture/terraform: {', '.join(missing)}")
|
||||
|
||||
# Check Mermaid diagram
|
||||
if re.search(r"\b(flowchart|graph)\b", mermaid_diagram):
|
||||
score += 1.0
|
||||
else:
|
||||
feedback.append("Invalid or missing Mermaid diagram syntax.")
|
||||
|
||||
# Check Terraform code
|
||||
if "resource" in terraform_code and "google_" in terraform_code:
|
||||
score += 1.0
|
||||
else:
|
||||
feedback.append("Terraform code does not contain Google Cloud resources.")
|
||||
|
||||
return {"score": score, "max_score": max_score, "feedback": feedback}
|
||||
|
||||
|
||||
def evaluate_packaging_phase(solution_guide: str, rubric: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Score Phase 3 Guide Packaging output."""
|
||||
score = 0.0
|
||||
max_score = 1.0
|
||||
feedback = []
|
||||
|
||||
required_sections = rubric.get("requires_guide_sections", [])
|
||||
found_sections = [sec for sec in required_sections if sec in solution_guide]
|
||||
|
||||
if len(found_sections) == len(required_sections):
|
||||
score += 1.0
|
||||
else:
|
||||
missing = set(required_sections) - set(found_sections)
|
||||
feedback.append(f"Missing required sections in solution guide: {', '.join(missing)}")
|
||||
|
||||
return {"score": score, "max_score": max_score, "feedback": feedback}
|
||||
|
||||
|
||||
def evaluate_case_run(final_state: Dict[str, Any], case: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Run full evaluation suite for a benchmark case output."""
|
||||
rubric = case.get("rubric", {})
|
||||
|
||||
req_eval = evaluate_requirements_phase(final_state.get("requirements_doc", ""))
|
||||
des_eval = evaluate_design_phase(
|
||||
final_state.get("architecture_doc", ""),
|
||||
final_state.get("mermaid_diagram", ""),
|
||||
final_state.get("terraform_code", ""),
|
||||
rubric,
|
||||
)
|
||||
pkg_eval = evaluate_packaging_phase(final_state.get("solution_guide", ""), rubric)
|
||||
|
||||
total_score = req_eval["score"] + des_eval["score"] + pkg_eval["score"]
|
||||
total_max = req_eval["max_score"] + des_eval["max_score"] + pkg_eval["max_score"]
|
||||
percentage = (total_score / total_max) * 100.0
|
||||
|
||||
all_feedback = req_eval["feedback"] + des_eval["feedback"] + pkg_eval["feedback"]
|
||||
|
||||
return {
|
||||
"case_id": case.get("id"),
|
||||
"case_name": case.get("name"),
|
||||
"total_score": total_score,
|
||||
"max_score": total_max,
|
||||
"percentage": percentage,
|
||||
"passed": percentage >= 80.0,
|
||||
"feedback": all_feedback,
|
||||
}
|
||||
58
eval/optimizer.py
Normal file
58
eval/optimizer.py
Normal file
@@ -0,0 +1,58 @@
|
||||
"""Skill Prompt Optimizer framework for tuning SKILL.md instruction prompts."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List
|
||||
from eval.metrics import evaluate_case_run
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SkillOptimizer:
|
||||
"""Analyzes benchmark evaluation feedback and recommends skill prompt adjustments."""
|
||||
|
||||
def generate_tuning_recommendations(self, eval_summary: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Generate prompt tuning recommendations based on evaluation failures."""
|
||||
recommendations = []
|
||||
results = eval_summary.get("results", [])
|
||||
|
||||
for res in results:
|
||||
if not res.get("passed"):
|
||||
feedback_items = res.get("feedback", [])
|
||||
for feedback in feedback_items:
|
||||
rec = self._map_feedback_to_skill_tuning(res.get("case_id"), feedback)
|
||||
if rec:
|
||||
recommendations.append(rec)
|
||||
|
||||
return recommendations
|
||||
|
||||
def _map_feedback_to_skill_tuning(self, case_id: str, feedback: str) -> Dict[str, Any] | None:
|
||||
"""Maps specific feedback strings to target SKILL.md prompt enhancements."""
|
||||
if "deferred" in feedback.lower():
|
||||
return {
|
||||
"target_skill": "requirements_discovery",
|
||||
"phase": "discover",
|
||||
"issue": feedback,
|
||||
"recommendation": "Emphasize product selection deferral rule in app/skills/requirements_discovery/SKILL.md frontmatter and guidelines.",
|
||||
}
|
||||
elif "mermaid" in feedback.lower():
|
||||
return {
|
||||
"target_skill": "architecture_design",
|
||||
"phase": "design",
|
||||
"issue": feedback,
|
||||
"recommendation": "Add strict Mermaid diagram syntax validation guidelines to app/skills/architecture_design/SKILL.md.",
|
||||
}
|
||||
elif "terraform" in feedback.lower():
|
||||
return {
|
||||
"target_skill": "architecture_design",
|
||||
"phase": "design",
|
||||
"issue": feedback,
|
||||
"recommendation": "Ensure Terraform provider and Google Cloud resource block patterns are explicit in app/skills/architecture_design/SKILL.md.",
|
||||
}
|
||||
elif "section" in feedback.lower():
|
||||
return {
|
||||
"target_skill": "packaging_guide",
|
||||
"phase": "package",
|
||||
"issue": feedback,
|
||||
"recommendation": "Include explicit section headers checklist in app/skills/packaging_guide/SKILL.md.",
|
||||
}
|
||||
return None
|
||||
32
eval/test_eval_harness.py
Normal file
32
eval/test_eval_harness.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""Tests for Evaluation Harness and Metrics."""
|
||||
|
||||
import pytest
|
||||
from eval.eval_harness import EvalHarness
|
||||
from eval.optimizer import SkillOptimizer
|
||||
|
||||
|
||||
def test_eval_harness_benchmark():
|
||||
"""Run EvalHarness against benchmark cases."""
|
||||
harness = EvalHarness()
|
||||
summary = harness.run_eval_suite()
|
||||
|
||||
assert summary["total_cases"] >= 1
|
||||
assert summary["passed_cases"] >= 1
|
||||
assert summary["pass_rate_percentage"] == 100.0
|
||||
|
||||
|
||||
def test_skill_optimizer_recommendations():
|
||||
"""Verify optimizer generates tuning recommendations for simulated failures."""
|
||||
optimizer = SkillOptimizer()
|
||||
simulated_summary = {
|
||||
"results": [
|
||||
{
|
||||
"case_id": "test-case",
|
||||
"passed": False,
|
||||
"feedback": ["Product selection was not explicitly deferred in discovery phase."],
|
||||
}
|
||||
]
|
||||
}
|
||||
recs = optimizer.generate_tuning_recommendations(simulated_summary)
|
||||
assert len(recs) == 1
|
||||
assert recs[0]["target_skill"] == "requirements_discovery"
|
||||
@@ -0,0 +1,5 @@
|
||||
pytest>=8.0.0
|
||||
pytest-asyncio>=0.23.0
|
||||
ruff>=0.6.0
|
||||
mypy>=1.11.0
|
||||
types-pyyaml>=6.0.0
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
langchain>=0.2.0
|
||||
langchain-core>=0.2.0
|
||||
langchain-openai>=0.1.0
|
||||
langgraph>=0.1.0
|
||||
starlette>=0.37.0
|
||||
uvicorn>=0.30.0
|
||||
pydantic>=2.7.0
|
||||
pydantic-settings>=2.2.0
|
||||
pyyaml>=6.0
|
||||
httpx>=0.27.0
|
||||
click>=8.1.0
|
||||
|
||||
@@ -4,12 +4,13 @@
|
||||
This guide packages manifest steps 0 through 7 for `gcp_solution_architecture_agent`, derived from the workflow-agent template. It is a reference baseline because no application-specific workflow request was supplied. Product selection was deferred during discovery and then made explicitly from the documented assumptions.
|
||||
|
||||
## Requirements
|
||||
See [`docs/requirements.md`](docs/requirements.md). It contains functional requirements, non-functional requirements, constraints, assumptions, and open questions. The principal unresolved items are identity, scale/SLOs, data model, edge exposure, compliance, DR, and delivery governance.
|
||||
See [`docs/requirements.md`](docs/requirements.md). It contains Functional requirements, Non-functional requirements, constraints, assumptions, and open questions. The principal unresolved items are identity, scale/SLOs, data model, edge exposure, compliance, DR, and delivery governance.
|
||||
|
||||
## Selected products
|
||||
Cloud Run, Pub/Sub, Cloud Storage, Firestore, VPC, Serverless VPC Access, Cloud Logging, Cloud Monitoring, Artifact Registry, IAM, and Service Usage.
|
||||
|
||||
## Architecture
|
||||
Mermaid Diagram:
|
||||
```mermaid
|
||||
flowchart LR
|
||||
C[External clients] -->|HTTPS| R[Cloud Run service]
|
||||
@@ -27,6 +28,7 @@ flowchart LR
|
||||
Clients use the HTTPS service. The service persists structured state and objects, emits events, and relies on a separate idempotent worker for asynchronous processing. The runtime has a dedicated identity, controlled egress, and managed encryption defaults. Public invocation is a deliberate baseline pending the ingress and identity answers in the open questions.
|
||||
|
||||
## Infrastructure as code
|
||||
Terraform configuration:
|
||||
The deployable Terraform root is in [`terraform/`](terraform/). It enables APIs, creates a custom VPC/subnet and serverless connector, runtime service account, uniformly private object bucket, Firestore database, event/dead-letter topics, Artifact Registry repository, and Cloud Run service. Variables keep project, region, image, and labels configurable. No credentials or backend state are committed.
|
||||
|
||||
Apply only after review:
|
||||
@@ -37,9 +39,9 @@ terraform -chdir=terraform apply
|
||||
```
|
||||
|
||||
## Validation
|
||||
See [`validation-results.md`](validation-results.md). The repository supplies formatting, initialization without a backend, Terraform validation, an opt-in refresh-free plan, and Python artifact tests. Resource deployment is not part of validation. In this generation environment, these commands are **not_run** because required executables, provider downloads, and credentials are unavailable; CI must run them and record the results.
|
||||
See [`validation-results.md`](validation-results.md) for full Validation results. The repository supplies formatting, initialization without a backend, Terraform validation, an opt-in refresh-free plan, and Python artifact tests. Resource deployment is not part of validation. In this generation environment, these commands are **not_run** because required executables, provider downloads, and credentials are unavailable; CI must run them and record the results.
|
||||
|
||||
## Repository verification
|
||||
## Repository Verification
|
||||
- Step 4 guide persistence: represented by this non-empty file at the required path.
|
||||
- Step 5 template/workflow conformance: represented by `workflow.yaml`, four phase entries, Terraform, diagram, requirements, and validation artifacts.
|
||||
- Step 6 publication: performed by the repository generation commit.
|
||||
|
||||
BIN
tests/__pycache__/__init__.cpython-312.pyc
Normal file
BIN
tests/__pycache__/__init__.cpython-312.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/conftest.cpython-312-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/conftest.cpython-312-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_agent_api.cpython-312-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_agent_api.cpython-312-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_artifacts.cpython-312-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_artifacts.cpython-312-pytest-9.1.1.pyc
Normal file
Binary file not shown.
Binary file not shown.
BIN
tests/__pycache__/test_package.cpython-312-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_package.cpython-312-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_skill_loader.cpython-312-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_skill_loader.cpython-312-pytest-9.1.1.pyc
Normal file
Binary file not shown.
52
tests/test_agent_api.py
Normal file
52
tests/test_agent_api.py
Normal file
@@ -0,0 +1,52 @@
|
||||
"""Starlette REST API integration tests using TestClient."""
|
||||
|
||||
import pytest
|
||||
from starlette.testclient import TestClient
|
||||
from app.agent import app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_health_endpoint(client):
|
||||
"""GET /health returns healthy status."""
|
||||
response = client.get("/health")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "healthy"
|
||||
assert "gcp_solution_architecture_agent" in data["agent"]
|
||||
|
||||
|
||||
def test_card_endpoint(client):
|
||||
"""GET /card returns agent capability card."""
|
||||
response = client.get("/card")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["name"] == "gcp_solution_architecture_agent"
|
||||
assert "discover" in data["capabilities"]["phases"]
|
||||
|
||||
|
||||
def test_generate_endpoint(client):
|
||||
"""POST /generate executes full agent workflow."""
|
||||
payload = {
|
||||
"request": "Build an event-driven regional ingestion service",
|
||||
"target_dir": ".",
|
||||
}
|
||||
response = client.post("/generate", json=payload)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "success"
|
||||
assert data["validation_passed"] is True
|
||||
assert "artifacts" in data
|
||||
assert "solution_guide" in data["artifacts"]
|
||||
|
||||
|
||||
def test_validate_endpoint(client):
|
||||
"""POST /validate executes pre-deployment validation checks."""
|
||||
payload = {"target_dir": "."}
|
||||
response = client.post("/validate", json=payload)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["valid"] is True
|
||||
32
tests/test_graph_workflow.py
Normal file
32
tests/test_graph_workflow.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""Integration tests for LangGraph StateGraph execution."""
|
||||
|
||||
import pytest
|
||||
from app.workflows.gcp_architecture_graph import create_agent
|
||||
from app.skills.loader import SkillLoader
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
def test_agent_graph_execution():
|
||||
"""Verify execution of the 4-phase LangGraph StateGraph."""
|
||||
settings = get_settings()
|
||||
loader = SkillLoader(settings.SKILLS_DIR)
|
||||
loader.load_skills()
|
||||
|
||||
agent = create_agent(loader)
|
||||
initial_state = {
|
||||
"workflow_request": "Event-driven regional HTTP application",
|
||||
"target_dir": ".",
|
||||
"active_skills": [],
|
||||
}
|
||||
|
||||
final_state = agent.invoke(initial_state)
|
||||
|
||||
assert final_state.get("current_phase") == "package"
|
||||
assert final_state.get("requirements_doc") is not None
|
||||
assert final_state.get("architecture_doc") is not None
|
||||
assert final_state.get("mermaid_diagram") is not None
|
||||
assert final_state.get("terraform_code") is not None
|
||||
assert final_state.get("validation_results") is not None
|
||||
assert final_state.get("solution_guide") is not None
|
||||
assert final_state.get("validation_passed") is True
|
||||
assert len(final_state.get("active_skills", [])) == 4
|
||||
45
tests/test_skill_loader.py
Normal file
45
tests/test_skill_loader.py
Normal file
@@ -0,0 +1,45 @@
|
||||
"""Unit tests for Local Dynamic Skill Loader."""
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from app.skills.loader import SkillLoader, LocalSkill
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
def test_skill_loader_discovery():
|
||||
"""Verify SkillLoader discovers all 4 phase skills under app/skills/."""
|
||||
settings = get_settings()
|
||||
loader = SkillLoader(settings.SKILLS_DIR)
|
||||
skills = loader.load_skills()
|
||||
|
||||
assert len(skills) >= 4
|
||||
assert "requirements_discovery" in skills
|
||||
assert "architecture_design" in skills
|
||||
assert "validation_rules" in skills
|
||||
assert "packaging_guide" in skills
|
||||
|
||||
|
||||
def test_skill_loader_by_phase():
|
||||
"""Verify filtering skills by phase."""
|
||||
settings = get_settings()
|
||||
loader = SkillLoader(settings.SKILLS_DIR)
|
||||
loader.load_skills()
|
||||
|
||||
discover_skills = loader.get_skills_by_phase("discover")
|
||||
assert len(discover_skills) == 1
|
||||
assert discover_skills[0].name == "requirements_discovery"
|
||||
|
||||
design_skills = loader.get_skills_by_phase("design")
|
||||
assert len(design_skills) == 1
|
||||
assert design_skills[0].name == "architecture_design"
|
||||
|
||||
|
||||
def test_format_skills_for_prompt():
|
||||
"""Verify formatting skill content into prompt text."""
|
||||
settings = get_settings()
|
||||
loader = SkillLoader(settings.SKILLS_DIR)
|
||||
loader.load_skills()
|
||||
|
||||
prompt = loader.format_skills_for_prompt("discover")
|
||||
assert "## Injected Skill Instructions" in prompt
|
||||
assert "requirements_discovery" in prompt
|
||||
Reference in New Issue
Block a user