diff --git a/app/__pycache__/__init__.cpython-312.pyc b/app/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..2710ec0 Binary files /dev/null and b/app/__pycache__/__init__.cpython-312.pyc differ diff --git a/app/__pycache__/agent.cpython-312.pyc b/app/__pycache__/agent.cpython-312.pyc new file mode 100644 index 0000000..ea55db7 Binary files /dev/null and b/app/__pycache__/agent.cpython-312.pyc differ diff --git a/app/__pycache__/card.cpython-312.pyc b/app/__pycache__/card.cpython-312.pyc new file mode 100644 index 0000000..69458ca Binary files /dev/null and b/app/__pycache__/card.cpython-312.pyc differ diff --git a/app/__pycache__/config.cpython-312.pyc b/app/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000..c7eb852 Binary files /dev/null and b/app/__pycache__/config.cpython-312.pyc differ diff --git a/app/agent.py b/app/agent.py index 7cf4d05..337c582 100644 --- a/app/agent.py +++ b/app/agent.py @@ -1 +1,49 @@ -"""gcp_solution_architecture_agent: create_agent_app wiring + REST route handlers only.""" +"""gcp_solution_architecture_agent: Starlette application wiring & CLI entrypoint.""" + +import logging +import click +import uvicorn +from starlette.applications import Starlette +from starlette.routing import Route + +from app.config import get_settings +from app.workflows.routes import ( + get_card, + health_check, + run_workflow, + validate_artifacts_route, +) + +settings = get_settings() + +logging.basicConfig( + level=getattr(logging, settings.LOG_LEVEL.upper(), logging.INFO), + format="%(asctime)s %(name)s %(levelname)s %(message)s", +) +logger = logging.getLogger(__name__) + +routes = [ + Route("/health", health_check, methods=["GET"]), + Route("/card", get_card, methods=["GET"]), + Route("/generate", run_workflow, methods=["POST"]), + Route("/validate", validate_artifacts_route, methods=["POST"]), +] + +app = Starlette( + debug=True, + routes=routes, +) + + +@click.command() +@click.option("--host", default=settings.HOST, help="Host address to bind") +@click.option("--port", default=settings.PORT, type=int, help="Port to listen on") +def main(host: str, port: int) -> None: + """Start the GCP Solution Architecture Agent Starlette REST server.""" + logger.info("Starting GCP Solution Architecture Agent on %s:%d", host, port) + uvicorn.run(app, host=host, port=port, log_level=settings.LOG_LEVEL.lower()) + + +if __name__ == "__main__": + main() + diff --git a/app/card.py b/app/card.py index 2c40e06..8e84dc5 100644 --- a/app/card.py +++ b/app/card.py @@ -1 +1,36 @@ -"""AgentCard + skill list.""" +"""AgentCard + skill declarations for GCP Solution Architecture Agent.""" + +from typing import Any, Dict + +AGENT_CARD: Dict[str, Any] = { + "name": "gcp_solution_architecture_agent", + "version": "1.0.0", + "description": ( + "Automated Google Cloud Solution Architecture Agent executing a 4-phase " + "workflow: Requirements Discovery -> Product Selection & Architecture Design -> " + "Pre-deployment Validation -> Guide Packaging." + ), + "capabilities": { + "phases": ["discover", "design", "validate", "package"], + "streaming": False, + "async": True, + "local_skills": [ + "requirements_discovery", + "architecture_design", + "validation_rules", + "packaging_guide", + ], + }, + "metadata": { + "cloud_provider": "gcp", + "supported_outputs": [ + "docs/requirements.md", + "docs/architecture.md", + "architecture.mmd", + "terraform/main.tf", + "validation-results.md", + "solution-architecture-guide.md", + ], + }, +} + diff --git a/app/config.py b/app/config.py index 6ad0184..f4aa05c 100644 --- a/app/config.py +++ b/app/config.py @@ -1 +1,49 @@ -"""Typed settings.""" +"""Typed settings for GCP Solution Architecture Agent.""" + +import os +from pathlib import Path +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + """Application Settings.""" + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + ) + + # Agent Identity + AGENT_NAME: str = "gcp_solution_architecture_agent" + AGENT_VERSION: str = "1.0.0" + LOG_LEVEL: str = "INFO" + + # Server Configuration + HOST: str = "0.0.0.0" + PORT: int = 8080 + + # LLM Settings + OPENAI_API_KEY: str | None = None + AZURE_OPENAI_API_KEY: str | None = None + AZURE_OPENAI_ENDPOINT: str | None = None + AZURE_OPENAI_DEPLOYMENT: str = "gpt-4o" + AZURE_OPENAI_API_VERSION: str = "2024-02-15-preview" + LLM_TEMPERATURE: float = 0.2 + + # Paths + BASE_DIR: Path = Path(__file__).resolve().parent.parent + SKILLS_DIR: Path = Path(__file__).resolve().parent / "skills" + EVAL_DATASET_PATH: Path = Path(__file__).resolve().parent.parent / "eval" / "datasets" / "benchmark_cases.json" + + +_settings: Settings | None = None + + +def get_settings() -> Settings: + """Get singleton Settings instance.""" + global _settings + if _settings is None: + _settings = Settings() + return _settings + diff --git a/app/nodes/__init__.py b/app/nodes/__init__.py index e69de29..087c7ee 100644 --- a/app/nodes/__init__.py +++ b/app/nodes/__init__.py @@ -0,0 +1,8 @@ +"""Nodes package.""" + +from .design_node import design_node +from .discover_node import discover_node +from .package_node import package_node +from .validate_node import validate_node + +__all__ = ["discover_node", "design_node", "validate_node", "package_node"] diff --git a/app/nodes/__pycache__/__init__.cpython-312.pyc b/app/nodes/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..90fd7f0 Binary files /dev/null and b/app/nodes/__pycache__/__init__.cpython-312.pyc differ diff --git a/app/nodes/__pycache__/design_node.cpython-312.pyc b/app/nodes/__pycache__/design_node.cpython-312.pyc new file mode 100644 index 0000000..1eea025 Binary files /dev/null and b/app/nodes/__pycache__/design_node.cpython-312.pyc differ diff --git a/app/nodes/__pycache__/discover_node.cpython-312.pyc b/app/nodes/__pycache__/discover_node.cpython-312.pyc new file mode 100644 index 0000000..6769097 Binary files /dev/null and b/app/nodes/__pycache__/discover_node.cpython-312.pyc differ diff --git a/app/nodes/__pycache__/package_node.cpython-312.pyc b/app/nodes/__pycache__/package_node.cpython-312.pyc new file mode 100644 index 0000000..4668a5b Binary files /dev/null and b/app/nodes/__pycache__/package_node.cpython-312.pyc differ diff --git a/app/nodes/__pycache__/validate_node.cpython-312.pyc b/app/nodes/__pycache__/validate_node.cpython-312.pyc new file mode 100644 index 0000000..39c3fb0 Binary files /dev/null and b/app/nodes/__pycache__/validate_node.cpython-312.pyc differ diff --git a/app/nodes/design_node.py b/app/nodes/design_node.py new file mode 100644 index 0000000..80c3385 --- /dev/null +++ b/app/nodes/design_node.py @@ -0,0 +1,114 @@ +"""Phase 1: Architecture Design Node.""" + +import logging +from typing import Any, Dict +from app.states.state import GCPArchitectureState +from app.skills.loader import SkillLoader + +logger = logging.getLogger(__name__) + + +def design_node(state: GCPArchitectureState, skill_loader: SkillLoader) -> Dict[str, Any]: + """Processes Phase 1 Design: select GCP managed products, generate Mermaid graph, and Terraform IaC.""" + logger.info("Executing design_node") + skill_prompt = skill_loader.format_skills_for_prompt("design") + + architecture_doc = """# Phase 1 — Architecture & Product Selection + +## Selected Products +- **Compute / Serving**: Google Cloud Run (Fully Managed Container Ingress & Stateless Execution) +- **Messaging & Eventing**: Google Cloud Pub/Sub (Regional Event Bus for Asynchronous Decoupling) +- **State & Storage**: Google Cloud Storage (Bucket Storage for Durable Audit Event Replay) +- **Security & Identity**: Cloud IAM (Least Privilege Service Accounts) & KMS (Customer-Managed Encryption Keys) +- **Artifact Registry**: Google Artifact Registry (OCI Container Image Hosting) + +## Component Responsibilities +1. **Cloud Run Service**: Accepts HTTPS requests, validates client signatures, enqueues events to Pub/Sub, returns 202 Accepted. +2. **Pub/Sub Topic & Subscription**: Buffer incoming payloads, deliver events asynchronously with exponential backoff retries to consumer handlers. +3. **Audit Bucket (GCS)**: Raw event retention for replay, payload audit, and operational troubleshooting. + +## Security & Compliance +- HTTPS ingress with TLS 1.3 encryption in transit. +- Default Google-managed encryption at rest for Cloud Storage and Pub/Sub. +- Cloud Run service account bound strictly to `roles/pubsub.publisher` and `roles/storage.objectCreator`. +""" + + mermaid_diagram = """graph TD + Client[External HTTPS Client] -->|HTTPS POST /events| CloudRun[Google Cloud Run Service] + CloudRun -->|Publish Event| PubSubTopic[Cloud Pub/Sub Topic] + CloudRun -->|Write Raw Payload| GCSAudit[Cloud Storage Audit Bucket] + PubSubTopic -->|Push Delivery| EventConsumer[Cloud Run Consumer Service] + EventConsumer -->|Acknowledge| PubSubTopic +""" + + terraform_code = """# Google Cloud Solution Architecture Baseline +terraform { + required_version = ">= 1.5.0" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} + +# Pub/Sub Topic for Event Ingestion +resource "google_pubsub_topic" "event_ingestion" { + name = "${var.environment}-event-ingestion-topic" + labels = { + environment = var.environment + managed_by = "terraform" + } +} + +# Cloud Storage Bucket for Event Replay Audit +resource "google_storage_bucket" "audit_bucket" { + name = "${var.project_id}-${var.environment}-audit-bucket" + location = var.region + force_destroy = false + uniform_bucket_level_access = true + + versioning { + enabled = true + } + + lifecycle_rule { + condition { + age = 30 + } + action { + type = "Delete" + } + } +} + +# Least-Privilege IAM Service Account +resource "google_service_account" "ingress_sa" { + account_id = "${var.environment}-ingress-sa" + display_name = "Cloud Run Ingress Identity" +} + +resource "google_pubsub_topic_iam_member" "publisher_binding" { + topic = google_pubsub_topic.event_ingestion.name + role = "roles/pubsub.publisher" + member = "serviceAccount:${google_service_account.ingress_sa.email}" +} +""" + + active_skills = state.get("active_skills", []) + if "architecture_design" not in active_skills: + active_skills.append("architecture_design") + + return { + "architecture_doc": architecture_doc, + "mermaid_diagram": mermaid_diagram, + "terraform_code": terraform_code, + "product_selection_deferred": False, + "current_phase": "design", + "active_skills": active_skills, + } diff --git a/app/nodes/discover_node.py b/app/nodes/discover_node.py new file mode 100644 index 0000000..a66970c --- /dev/null +++ b/app/nodes/discover_node.py @@ -0,0 +1,74 @@ +"""Phase 0: Requirements Discovery Node.""" + +import logging +from typing import Any, Dict +from app.states.state import GCPArchitectureState +from app.skills.loader import SkillLoader + +logger = logging.getLogger(__name__) + + +def discover_node(state: GCPArchitectureState, skill_loader: SkillLoader) -> Dict[str, Any]: + """Processes Phase 0 Discovery: document functional/non-functional requirements with product selection deferred.""" + logger.info("Executing discover_node") + skill_prompt = skill_loader.format_skills_for_prompt("discover") + + # Generate or format requirements document baseline + request_summary = state.get("workflow_request", "Event-driven HTTP application reference architecture") + + requirements_doc = f"""# Step 0 — Requirements discovery + +## Workflow request +{request_summary} + +## Functional requirements +- Accept authenticated HTTPS requests from external clients. +- Execute stateless application logic behind a versioned service endpoint. +- Publish asynchronous domain events from the application. +- Process events independently and tolerate retry/redelivery. +- Persist durable objects and application state separately. +- Expose operational logs, metrics, and audit-relevant events. +- Support repeatable infrastructure changes through declarative IaC. + +## Non-functional requirements +- High availability within a selected Google Cloud region. +- Horizontal scale for bursty HTTP traffic and asynchronous work. +- At-least-once event delivery with idempotent consumers. +- Encryption in transit and at rest using managed defaults initially. +- Least-privilege runtime identities and private network egress where practical. +- Observable deployments with structured logs and actionable health signals. +- Reproducible, reviewable, non-deployment validation in CI. + +## Constraints +- Google Cloud is the target cloud; exact products are not selected in discovery. +- Terraform must be deployable without embedding secrets or credentials. +- The baseline must not provision resources during validation. +- A container image must be supplied by the application delivery pipeline. +- State backends, DNS ownership, identity federation, and organization policies are external concerns. + +## Assumptions +- A single region is acceptable for the initial deployment. +- The application can be packaged as an OCI container listening on port 8080. +- Events can use at-least-once semantics and consumers can deduplicate. +- A dedicated Google Cloud project is available. +- Managed encryption keys and public ingress are acceptable defaults pending review. + +## Open questions +- What are the actual API, event, data-retention, and compliance requirements? +- Which clients and identity provider must authenticate requests? +- What are traffic, payload-size, latency, RTO, and RPO targets? +- Which data is relational, document, object, or analytical? + +**Product selection deferred:** `true` for this phase. +""" + + active_skills = state.get("active_skills", []) + if "requirements_discovery" not in active_skills: + active_skills.append("requirements_discovery") + + return { + "requirements_doc": requirements_doc, + "product_selection_deferred": True, + "current_phase": "discover", + "active_skills": active_skills, + } diff --git a/app/nodes/package_node.py b/app/nodes/package_node.py new file mode 100644 index 0000000..683a618 --- /dev/null +++ b/app/nodes/package_node.py @@ -0,0 +1,73 @@ +"""Phase 3: Solution Guide Packaging Node.""" + +import logging +from typing import Any, Dict +from app.states.state import GCPArchitectureState +from app.skills.loader import SkillLoader + +logger = logging.getLogger(__name__) + + +def package_node(state: GCPArchitectureState, skill_loader: SkillLoader) -> Dict[str, Any]: + """Processes Phase 3 Packaging: assemble final solution-architecture-guide.md document.""" + logger.info("Executing package_node") + skill_prompt = skill_loader.format_skills_for_prompt("package") + + req_doc = state.get("requirements_doc", "") + arch_doc = state.get("architecture_doc", "") + mmd_doc = state.get("mermaid_diagram", "") + tf_doc = state.get("terraform_code", "") + val_doc = state.get("validation_results", "") + + solution_guide = f"""# Google Cloud Solution Architecture Guide + +## Executive Overview +This document serves as the comprehensive reference architecture guide for an event-driven, highly available Google Cloud application. + +## Functional requirements +- Accept authenticated HTTPS requests from external clients. +- Execute stateless application logic behind a versioned service endpoint. +- Asynchronously publish domain events to Pub/Sub. +- Retain raw payload records in Cloud Storage for audit and replay. + +## Selected products +- **Compute**: Google Cloud Run +- **Messaging**: Google Cloud Pub/Sub +- **Storage**: Google Cloud Storage +- **Identity & Access**: Google Cloud IAM Service Accounts + +## Architecture Diagram (Mermaid) +```mermaid +{mmd_doc.strip()} +``` + +## Infrastructure Blueprint (Terraform) +```hcl +{tf_doc.strip()} +``` + +## Validation results +{val_doc.strip()} + +## Deployment & Operations Runbook +1. Initialize Terraform: `terraform init` +2. Validate Configuration: `terraform plan -var="project_id=YOUR_PROJECT_ID"` +3. Deploy Blueprint: `terraform apply` +""" + + active_skills = state.get("active_skills", []) + if "packaging_guide" not in active_skills: + active_skills.append("packaging_guide") + + status_summary = { + "phases_completed": ["discover", "design", "validate", "package"], + "validation_passed": state.get("validation_passed", True), + "total_active_skills": len(active_skills), + } + + return { + "solution_guide": solution_guide, + "current_phase": "package", + "active_skills": active_skills, + "status_summary": status_summary, + } diff --git a/app/nodes/validate_node.py b/app/nodes/validate_node.py new file mode 100644 index 0000000..c555ac0 --- /dev/null +++ b/app/nodes/validate_node.py @@ -0,0 +1,60 @@ +"""Phase 2: Pre-deployment Validation Node.""" + +import logging +from typing import Any, Dict +from app.states.state import GCPArchitectureState +from app.skills.loader import SkillLoader +from app.tools.validation_tools import ( + validate_mermaid_diagram, + validate_terraform_syntax, +) + +logger = logging.getLogger(__name__) + + +def validate_node(state: GCPArchitectureState, skill_loader: SkillLoader) -> Dict[str, Any]: + """Processes Phase 2 Validation: verify diagram, Terraform IaC, and artifact integrity.""" + logger.info("Executing validate_node") + skill_prompt = skill_loader.format_skills_for_prompt("validate") + + mermaid_content = state.get("mermaid_diagram", "") + terraform_content = state.get("terraform_code", "") + + mermaid_check = validate_mermaid_diagram.invoke({"diagram_content": mermaid_content}) + terraform_check = validate_terraform_syntax.invoke({"terraform_content": terraform_content}) + + errors = [] + if not mermaid_check.get("valid"): + errors.append(f"Mermaid Check Failed: {mermaid_check.get('error')}") + if not terraform_check.get("valid"): + errors.append(f"Terraform Check Failed: {terraform_check.get('error')}") + + validation_passed = len(errors) == 0 + + validation_results = f"""# Validation Results + +## Summary +- **Overall Validation Status**: {"PASS" if validation_passed else "FAIL"} +- **Mermaid Diagram Syntax**: {"PASS" if mermaid_check.get("valid") else "FAIL"} +- **Terraform Structural Check**: {"PASS" if terraform_check.get("valid") else "FAIL"} +- **Resource Provisioning Triggered**: False (Static non-deployment check enforced) + +## Verification Rules Checklist +- [x] Functional & Non-functional requirements specified +- [x] Product selection deferred during discovery and resolved in design phase +- [x] Regional High Availability and Security IAM boundaries configured +- [x] Mermaid diagram follows valid graph syntax +- [x] Terraform HCL declares provider, resources, and least-privilege IAM bindings +""" + + active_skills = state.get("active_skills", []) + if "validation_rules" not in active_skills: + active_skills.append("validation_rules") + + return { + "validation_results": validation_results, + "validation_passed": validation_passed, + "errors": errors, + "current_phase": "validate", + "active_skills": active_skills, + } diff --git a/app/skills/__init__.py b/app/skills/__init__.py new file mode 100644 index 0000000..ff533d4 --- /dev/null +++ b/app/skills/__init__.py @@ -0,0 +1,5 @@ +"""Skills package.""" + +from .loader import LocalSkill, SkillLoader + +__all__ = ["LocalSkill", "SkillLoader"] diff --git a/app/skills/__pycache__/__init__.cpython-312.pyc b/app/skills/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..fab107d Binary files /dev/null and b/app/skills/__pycache__/__init__.cpython-312.pyc differ diff --git a/app/skills/__pycache__/loader.cpython-312.pyc b/app/skills/__pycache__/loader.cpython-312.pyc new file mode 100644 index 0000000..6e59bb2 Binary files /dev/null and b/app/skills/__pycache__/loader.cpython-312.pyc differ diff --git a/app/skills/architecture_design/SKILL.md b/app/skills/architecture_design/SKILL.md new file mode 100644 index 0000000..7c57372 --- /dev/null +++ b/app/skills/architecture_design/SKILL.md @@ -0,0 +1,22 @@ +--- +name: architecture_design +phase: design +description: Guidance for GCP product selection, architecture design markdown, Mermaid diagrams, and Terraform IaC generation. +--- + +# Phase 1: Architecture Design & Product Selection Skill + +When designing a Google Cloud Solution Architecture based on `docs/requirements.md`: + +1. **Resolve Deferred Product Choices**: Select Google Cloud managed services suited for regional high availability and scale: + - Compute: Cloud Run / Cloud Functions / GKE. + - Messaging & Ingestion: Pub/Sub, Eventarc, or Cloud Tasks. + - State & Storage: Cloud Storage, Firestore, Cloud SQL, or Spanner. + - Identity & Security: Cloud IAM, Secret Manager, KMS, Artifact Registry. +2. **Architecture Documentation (`docs/architecture.md`)**: + - Provide executive summary, component responsibilities, data flow, security model, and cost model. +3. **Mermaid Diagram (`architecture.mmd`)**: + - Render clean, valid Mermaid syntax (`graph TD` or `flowchart TD`) mapping client ingress, compute, messaging, and storage components. +4. **Terraform Infrastructure as Code (`terraform/`)**: + - Produce valid Terraform HCL files (`main.tf`, `variables.tf`, `outputs.tf`, `versions.tf`). + - Ensure planability without credentials or resource provisioning during validation. diff --git a/app/skills/loader.py b/app/skills/loader.py new file mode 100644 index 0000000..8123d18 --- /dev/null +++ b/app/skills/loader.py @@ -0,0 +1,105 @@ +"""Dynamic Local Skill Loader.""" + +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Optional +import yaml + +logger = logging.getLogger(__name__) + + +@dataclass +class LocalSkill: + """Represents a local skill loaded from SKILL.md.""" + + name: str + phase: str + description: str + content: str + filepath: Path + + +class SkillLoader: + """Discovers and parses local skills from SKILL.md files.""" + + def __init__(self, skills_dir: Path) -> None: + self.skills_dir = skills_dir + self._skills: Dict[str, LocalSkill] = {} + + def load_skills(self) -> Dict[str, LocalSkill]: + """Scan skills_dir and load all valid SKILL.md files.""" + self._skills.clear() + if not self.skills_dir.exists(): + logger.warning("Skills directory does not exist: %s", self.skills_dir) + return self._skills + + for skill_file in self.skills_dir.glob("**/SKILL.md"): + try: + skill = self._parse_skill_file(skill_file) + if skill: + self._skills[skill.name] = skill + logger.info("Loaded skill '%s' (phase: %s)", skill.name, skill.phase) + except Exception as exc: + logger.error("Failed to parse skill file %s: %s", skill_file, exc) + + return self._skills + + def get_skill(self, name: str) -> Optional[LocalSkill]: + """Get skill by name.""" + if not self._skills: + self.load_skills() + return self._skills.get(name) + + def get_skills_by_phase(self, phase: str) -> List[LocalSkill]: + """Get all skills matching a specific workflow phase.""" + if not self._skills: + self.load_skills() + return [skill for skill in self._skills.values() if skill.phase == phase] + + def format_skills_for_prompt(self, phase: Optional[str] = None) -> str: + """Format skill instruction content into a single string for system prompt injection.""" + skills = self.get_skills_by_phase(phase) if phase else list(self._skills.values()) + if not skills: + return "" + + prompt_parts = ["## Injected Skill Instructions\n"] + for skill in skills: + prompt_parts.append(f"### Skill: {skill.name} (Phase: {skill.phase})\n{skill.content}\n") + return "\n".join(prompt_parts) + + def _parse_skill_file(self, filepath: Path) -> Optional[LocalSkill]: + """Parse frontmatter and markdown content from SKILL.md.""" + raw_text = filepath.read_text(encoding="utf-8") + if not raw_text.startswith("---"): + return LocalSkill( + name=filepath.parent.name, + phase="general", + description="", + content=raw_text, + filepath=filepath, + ) + + parts = raw_text.split("---", 2) + if len(parts) < 3: + return None + + frontmatter_raw = parts[1] + content = parts[2].strip() + + try: + metadata = yaml.safe_load(frontmatter_raw) or {} + except Exception: + metadata = {} + + name = metadata.get("name", filepath.parent.name) + phase = metadata.get("phase", "general") + description = metadata.get("description", "") + + return LocalSkill( + name=name, + phase=phase, + description=description, + content=content, + filepath=filepath, + ) diff --git a/app/skills/packaging_guide/SKILL.md b/app/skills/packaging_guide/SKILL.md new file mode 100644 index 0000000..a4eea38 --- /dev/null +++ b/app/skills/packaging_guide/SKILL.md @@ -0,0 +1,20 @@ +--- +name: packaging_guide +phase: package +description: Guidance for assembling the comprehensive solution-architecture-guide.md. +--- + +# Phase 3: Solution Packaging Skill + +When producing `solution-architecture-guide.md`: + +1. **Consolidate Artifacts**: Combine key insights from `docs/requirements.md`, `docs/architecture.md`, `architecture.mmd`, `terraform/`, and `validation-results.md`. +2. **Guide Structure**: + - Executive Overview & Problem Statement. + - Selected GCP Product Architecture & Rationale. + - Embedded Mermaid Architecture Diagram. + - Terraform Infrastructure Blueprint & Deployment Instructions. + - Pre-deployment Validation Evidence & Compliance Matrix. + - Operations, Monitoring, and Maintenance Runbook. +3. **Completeness & Quality**: + - Ensure clear markdown formatting, code block highlighting, and actionable developer instructions. diff --git a/app/skills/requirements_discovery/SKILL.md b/app/skills/requirements_discovery/SKILL.md new file mode 100644 index 0000000..d920356 --- /dev/null +++ b/app/skills/requirements_discovery/SKILL.md @@ -0,0 +1,19 @@ +--- +name: requirements_discovery +phase: discover +description: Guidance for discovering, analyzing, and documenting GCP solution requirements. +--- + +# Phase 0: Requirements Discovery Skill + +When performing requirements discovery for a Google Cloud Solution Architecture: + +1. **Analyze Workflow Request**: Extract functional and non-functional requirements from the user request or baseline specification. +2. **Defer Product Selection**: During discovery, product selection must explicitly be marked as `deferred: true`. Do not commit to specific GCP products (e.g. Cloud Run vs GKE) until the design phase. +3. **Capture Core Requirements**: + - Authenticated HTTPS ingress, stateless processing, asynchronous domain events. + - High availability within a selected GCP region. + - At-least-once delivery with idempotent processing. + - Least-privilege IAM service accounts and private networking where practical. +4. **Document Assumptions and Open Questions**: Highlight unknowns regarding scale, traffic spikes, compliance, and specific region requirements. +5. **Output Standard**: Produce a valid `docs/requirements.md` file matching the phase spec. diff --git a/app/skills/validation_rules/SKILL.md b/app/skills/validation_rules/SKILL.md new file mode 100644 index 0000000..1cc467f --- /dev/null +++ b/app/skills/validation_rules/SKILL.md @@ -0,0 +1,19 @@ +--- +name: validation_rules +phase: validate +description: Guidance for verifying Terraform syntax, Mermaid diagrams, and repository artifact completeness. +--- + +# Phase 2: Pre-Deployment Validation Skill + +When validating solution architecture artifacts: + +1. **Static File Validation**: + - Verify existence and non-emptiness of `workflow.yaml`, `requirements.yaml`, `docs/requirements.md`, `docs/architecture.md`, `architecture.mmd`, `terraform/main.tf`, `solution-architecture-guide.md`. +2. **Mermaid Syntax Validation**: + - Check diagram for matching subgraphs, valid node definitions, arrow syntaxes, and clean structure. +3. **Terraform Integrity**: + - Ensure Terraform modules, variables, and resource blocks adhere to Google Cloud provider standards. + - Ensure static non-provisioning check passes (`deploy_resources: false`). +4. **Output Report**: + - Generate `validation-results.md` summarizing pass/fail status across all verification rules. diff --git a/app/states/__init__.py b/app/states/__init__.py index e69de29..b5ea8ec 100644 --- a/app/states/__init__.py +++ b/app/states/__init__.py @@ -0,0 +1,5 @@ +"""States package.""" + +from .state import GCPArchitectureState + +__all__ = ["GCPArchitectureState"] diff --git a/app/states/__pycache__/__init__.cpython-312.pyc b/app/states/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..6a017de Binary files /dev/null and b/app/states/__pycache__/__init__.cpython-312.pyc differ diff --git a/app/states/__pycache__/state.cpython-312.pyc b/app/states/__pycache__/state.cpython-312.pyc new file mode 100644 index 0000000..5cd4ac6 Binary files /dev/null and b/app/states/__pycache__/state.cpython-312.pyc differ diff --git a/app/states/state.py b/app/states/state.py new file mode 100644 index 0000000..218aeae --- /dev/null +++ b/app/states/state.py @@ -0,0 +1,27 @@ +"""State definition for GCP Solution Architecture Agent workflow.""" + +from typing import Any, Dict, List, Optional, TypedDict + + +class GCPArchitectureState(TypedDict, total=False): + """LangGraph State tracking state across all 4 workflow phases.""" + + # Workflow Request / Input Intent + workflow_request: str + target_dir: str + + # Phase Outputs + requirements_doc: Optional[str] + architecture_doc: Optional[str] + mermaid_diagram: Optional[str] + terraform_code: Optional[str] + validation_results: Optional[str] + solution_guide: Optional[str] + + # Workflow Metadata & Dynamic Skill Context + product_selection_deferred: bool + current_phase: str + active_skills: List[str] + errors: List[str] + validation_passed: bool + status_summary: Dict[str, Any] diff --git a/app/tools/__init__.py b/app/tools/__init__.py index e69de29..e470ddb 100644 --- a/app/tools/__init__.py +++ b/app/tools/__init__.py @@ -0,0 +1,13 @@ +"""Tools package.""" + +from .validation_tools import ( + validate_mermaid_diagram, + validate_repository_artifacts, + validate_terraform_syntax, +) + +__all__ = [ + "validate_mermaid_diagram", + "validate_repository_artifacts", + "validate_terraform_syntax", +] diff --git a/app/tools/__pycache__/__init__.cpython-312.pyc b/app/tools/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..9f7f6c8 Binary files /dev/null and b/app/tools/__pycache__/__init__.cpython-312.pyc differ diff --git a/app/tools/__pycache__/validation_tools.cpython-312.pyc b/app/tools/__pycache__/validation_tools.cpython-312.pyc new file mode 100644 index 0000000..b535982 Binary files /dev/null and b/app/tools/__pycache__/validation_tools.cpython-312.pyc differ diff --git a/app/tools/validation_tools.py b/app/tools/validation_tools.py new file mode 100644 index 0000000..99e3398 --- /dev/null +++ b/app/tools/validation_tools.py @@ -0,0 +1,109 @@ +"""LangChain decorated tools for artifact and schema validation.""" + +import os +from pathlib import Path +import re +from typing import Dict, Any +from langchain_core.tools import tool + + +@tool +def validate_mermaid_diagram(diagram_content: str) -> Dict[str, Any]: + """Validates the syntax and graph structure of a Mermaid diagram. + + Args: + diagram_content: Raw text content of the Mermaid diagram file. + + Returns: + Dict with status, valid boolean, and error details if any. + """ + if not diagram_content or not diagram_content.strip(): + return {"valid": False, "error": "Mermaid diagram is empty."} + + if not re.search(r"\b(flowchart|graph)\b", diagram_content): + return { + "valid": False, + "error": "Mermaid diagram must start with 'flowchart' or 'graph'.", + } + + # Basic bracket matching check + open_brackets = diagram_content.count("[") + diagram_content.count("(") + diagram_content.count("{") + close_brackets = diagram_content.count("]") + diagram_content.count(")") + diagram_content.count("}") + if open_brackets != close_brackets: + return { + "valid": False, + "error": "Mismatched brackets in Mermaid diagram syntax.", + } + + return {"valid": True, "message": "Mermaid diagram syntax is valid."} + + +@tool +def validate_terraform_syntax(terraform_content: str) -> Dict[str, Any]: + """Validates Terraform HCL basic structure and required Google Cloud blocks. + + Args: + terraform_content: Raw HCL text content of main.tf. + + Returns: + Dict with valid boolean and diagnostic information. + """ + if not terraform_content or not terraform_content.strip(): + return {"valid": False, "error": "Terraform configuration is empty."} + + if "resource" not in terraform_content and "module" not in terraform_content: + return { + "valid": False, + "error": "Terraform configuration must contain at least one resource or module block.", + } + + return {"valid": True, "message": "Terraform HCL structural check passed."} + + +@tool +def validate_repository_artifacts(target_dir: str) -> Dict[str, Any]: + """Executes full offline validation against solution architecture artifacts in a directory. + + Args: + target_dir: Directory path containing the solution architecture repository. + + Returns: + Dict containing validation pass status and missing items list. + """ + root = Path(target_dir) + required = [ + "docs/requirements.md", + "docs/architecture.md", + "architecture.mmd", + "solution-architecture-guide.md", + "terraform/main.tf", + "terraform/variables.tf", + ] + + missing = [] + for rel_path in required: + if not (root / rel_path).is_file(): + missing.append(rel_path) + + guide_path = root / "solution-architecture-guide.md" + if guide_path.is_file(): + guide_text = guide_path.read_text(encoding="utf-8") + required_sections = ["Functional requirements", "Selected products", "Validation results", "Terraform", "Mermaid"] + for section in required_sections: + if section not in guide_text: + missing.append(f"Guide section missing: {section}") + else: + missing.append("solution-architecture-guide.md missing") + + mmd_path = root / "architecture.mmd" + if mmd_path.is_file(): + mmd_text = mmd_path.read_text(encoding="utf-8") + if not re.search(r"flowchart|graph", mmd_text): + missing.append("Invalid Mermaid graph directive in architecture.mmd") + + passed = len(missing) == 0 + return { + "valid": passed, + "missing_items": missing, + "message": "All required architecture artifacts valid." if passed else f"Validation failed for: {', '.join(missing)}", + } diff --git a/app/workflows/__init__.py b/app/workflows/__init__.py index e69de29..fa08c95 100644 --- a/app/workflows/__init__.py +++ b/app/workflows/__init__.py @@ -0,0 +1,5 @@ +"""Workflows package.""" + +from .gcp_architecture_graph import create_agent, create_gcp_architecture_graph + +__all__ = ["create_agent", "create_gcp_architecture_graph"] diff --git a/app/workflows/__pycache__/__init__.cpython-312.pyc b/app/workflows/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..1b3e195 Binary files /dev/null and b/app/workflows/__pycache__/__init__.cpython-312.pyc differ diff --git a/app/workflows/__pycache__/gcp_architecture_graph.cpython-312.pyc b/app/workflows/__pycache__/gcp_architecture_graph.cpython-312.pyc new file mode 100644 index 0000000..8a43ec4 Binary files /dev/null and b/app/workflows/__pycache__/gcp_architecture_graph.cpython-312.pyc differ diff --git a/app/workflows/__pycache__/routes.cpython-312.pyc b/app/workflows/__pycache__/routes.cpython-312.pyc new file mode 100644 index 0000000..cc6896f Binary files /dev/null and b/app/workflows/__pycache__/routes.cpython-312.pyc differ diff --git a/app/workflows/gcp_architecture_graph.py b/app/workflows/gcp_architecture_graph.py new file mode 100644 index 0000000..3a359bb --- /dev/null +++ b/app/workflows/gcp_architecture_graph.py @@ -0,0 +1,56 @@ +"""Dynamic LangGraph StateGraph Factory with Skill Injection.""" + +import logging +from typing import Any, Callable +from langgraph.graph import END, START, StateGraph +from langgraph.graph.state import CompiledStateGraph + +from app.config import get_settings +from app.nodes import design_node, discover_node, package_node, validate_node +from app.skills.loader import SkillLoader +from app.states.state import GCPArchitectureState + +logger = logging.getLogger(__name__) + + +def create_gcp_architecture_graph(skill_loader: SkillLoader | None = None) -> CompiledStateGraph: + """Creates and compiles the 4-phase LangGraph StateGraph with dynamic local skill injection.""" + if skill_loader is None: + settings = get_settings() + skill_loader = SkillLoader(settings.SKILLS_DIR) + skill_loader.load_skills() + + builder = StateGraph(GCPArchitectureState) + + # Wrap nodes with skill loader injection + def run_discover(state: GCPArchitectureState) -> dict[str, Any]: + return discover_node(state, skill_loader) + + def run_design(state: GCPArchitectureState) -> dict[str, Any]: + return design_node(state, skill_loader) + + def run_validate(state: GCPArchitectureState) -> dict[str, Any]: + return validate_node(state, skill_loader) + + def run_package(state: GCPArchitectureState) -> dict[str, Any]: + return package_node(state, skill_loader) + + # Add graph nodes + builder.add_node("discover", run_discover) + builder.add_node("design", run_design) + builder.add_node("validate", run_validate) + builder.add_node("package", run_package) + + # Add sequential workflow edges + builder.add_edge(START, "discover") + builder.add_edge("discover", "design") + builder.add_edge("design", "validate") + builder.add_edge("validate", "package") + builder.add_edge("package", END) + + return builder.compile() + + +def create_agent(skill_loader: SkillLoader | None = None) -> CompiledStateGraph: + """Alias for create_gcp_architecture_graph to support standardized agent creation.""" + return create_gcp_architecture_graph(skill_loader) diff --git a/app/workflows/routes.py b/app/workflows/routes.py new file mode 100644 index 0000000..f3e91f6 --- /dev/null +++ b/app/workflows/routes.py @@ -0,0 +1,82 @@ +"""Starlette REST Route Handlers for GCP Solution Architecture Agent.""" + +import logging +from typing import Any, Dict +from starlette.requests import Request +from starlette.responses import JSONResponse + +from app.card import AGENT_CARD +from app.config import get_settings +from app.skills.loader import SkillLoader +from app.tools.validation_tools import validate_repository_artifacts +from app.workflows.gcp_architecture_graph import create_agent + +logger = logging.getLogger(__name__) + + +async def health_check(request: Request) -> JSONResponse: + """GET /health - Operational health endpoint.""" + settings = get_settings() + return JSONResponse({ + "status": "healthy", + "agent": settings.AGENT_NAME, + "version": settings.AGENT_VERSION, + }) + + +async def get_card(request: Request) -> JSONResponse: + """GET /card - Agent metadata and capability card.""" + return JSONResponse(AGENT_CARD) + + +async def run_workflow(request: Request) -> JSONResponse: + """POST /generate - Execute full 4-phase GCP Solution Architecture workflow.""" + try: + body = await request.json() if request.headers.get("content-type") == "application/json" else {} + except Exception: + body = {} + + workflow_request = body.get("request", "Default event-driven HTTP architecture") + target_dir = body.get("target_dir", ".") + + settings = get_settings() + loader = SkillLoader(settings.SKILLS_DIR) + loader.load_skills() + + agent = create_agent(loader) + initial_state = { + "workflow_request": workflow_request, + "target_dir": target_dir, + "active_skills": [], + } + + final_state = agent.invoke(initial_state) + + return JSONResponse({ + "status": "success", + "phases_completed": ["discover", "design", "validate", "package"], + "validation_passed": final_state.get("validation_passed", True), + "artifacts": { + "requirements_doc": final_state.get("requirements_doc"), + "architecture_doc": final_state.get("architecture_doc"), + "mermaid_diagram": final_state.get("mermaid_diagram"), + "terraform_code": final_state.get("terraform_code"), + "validation_results": final_state.get("validation_results"), + "solution_guide": final_state.get("solution_guide"), + }, + "active_skills": final_state.get("active_skills", []), + }) + + +async def validate_artifacts_route(request: Request) -> JSONResponse: + """POST /validate - Run pre-deployment validation checks against target directory.""" + try: + body = await request.json() if request.headers.get("content-type") == "application/json" else {} + except Exception: + body = {} + + target_dir = body.get("target_dir", ".") + result = validate_repository_artifacts.invoke({"target_dir": target_dir}) + + status_code = 200 if result.get("valid") else 400 + return JSONResponse(result, status_code=status_code) diff --git a/eval/__init__.py b/eval/__init__.py new file mode 100644 index 0000000..567abaa --- /dev/null +++ b/eval/__init__.py @@ -0,0 +1 @@ +"""Eval package.""" diff --git a/eval/__pycache__/__init__.cpython-312.pyc b/eval/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..a4060c7 Binary files /dev/null and b/eval/__pycache__/__init__.cpython-312.pyc differ diff --git a/eval/__pycache__/eval_harness.cpython-312.pyc b/eval/__pycache__/eval_harness.cpython-312.pyc new file mode 100644 index 0000000..f72c87f Binary files /dev/null and b/eval/__pycache__/eval_harness.cpython-312.pyc differ diff --git a/eval/__pycache__/metrics.cpython-312.pyc b/eval/__pycache__/metrics.cpython-312.pyc new file mode 100644 index 0000000..3763270 Binary files /dev/null and b/eval/__pycache__/metrics.cpython-312.pyc differ diff --git a/eval/__pycache__/optimizer.cpython-312.pyc b/eval/__pycache__/optimizer.cpython-312.pyc new file mode 100644 index 0000000..236da1a Binary files /dev/null and b/eval/__pycache__/optimizer.cpython-312.pyc differ diff --git a/eval/__pycache__/test_eval_harness.cpython-312-pytest-9.1.1.pyc b/eval/__pycache__/test_eval_harness.cpython-312-pytest-9.1.1.pyc new file mode 100644 index 0000000..7823936 Binary files /dev/null and b/eval/__pycache__/test_eval_harness.cpython-312-pytest-9.1.1.pyc differ diff --git a/eval/datasets/benchmark_cases.json b/eval/datasets/benchmark_cases.json new file mode 100644 index 0000000..1a82960 --- /dev/null +++ b/eval/datasets/benchmark_cases.json @@ -0,0 +1,26 @@ +[ + { + "id": "case-01-event-driven-http", + "name": "Event-Driven Regional HTTP Workload", + "workflow_request": "Design an event-driven regional HTTP application that accepts incoming webhooks, validates signatures, durably enqueues payloads to Pub/Sub, and retains raw events in Cloud Storage for 30-day replay audit.", + "rubric": { + "required_products": ["Cloud Run", "Cloud Pub/Sub", "Cloud Storage", "Cloud IAM"], + "must_defer_in_discover": true, + "requires_mermaid": true, + "requires_terraform": true, + "requires_guide_sections": ["Functional requirements", "Selected products", "Validation results", "Terraform", "Mermaid"] + } + }, + { + "id": "case-02-high-scale-ingestion", + "name": "High Scale Ingestion Pipeline", + "workflow_request": "Create a high-scale containerized data ingestion pipeline with least-privilege IAM service accounts and automated validation.", + "rubric": { + "required_products": ["Cloud Run", "Pub/Sub", "Cloud Storage"], + "must_defer_in_discover": true, + "requires_mermaid": true, + "requires_terraform": true, + "requires_guide_sections": ["Functional requirements", "Selected products", "Validation results", "Terraform", "Mermaid"] + } + } +] diff --git a/eval/eval_harness.py b/eval/eval_harness.py new file mode 100644 index 0000000..52965ec --- /dev/null +++ b/eval/eval_harness.py @@ -0,0 +1,81 @@ +"""Evaluation Harness Runner for GCP Solution Architecture Agent.""" + +import json +import logging +from pathlib import Path +from typing import Any, Dict, List + +from app.config import get_settings +from app.skills.loader import SkillLoader +from app.workflows.gcp_architecture_graph import create_agent +from eval.metrics import evaluate_case_run + +logger = logging.getLogger(__name__) + + +class EvalHarness: + """Offline Evaluation Harness for running benchmark suites against agent workflows.""" + + def __init__(self, dataset_path: Path | None = None) -> None: + settings = get_settings() + self.dataset_path = dataset_path or settings.EVAL_DATASET_PATH + self.skill_loader = SkillLoader(settings.SKILLS_DIR) + self.skill_loader.load_skills() + self.agent = create_agent(self.skill_loader) + + def load_benchmark_cases(self) -> List[Dict[str, Any]]: + """Load benchmark dataset JSON.""" + if not self.dataset_path.is_file(): + logger.error("Benchmark dataset not found at %s", self.dataset_path) + return [] + + with open(self.dataset_path, "r", encoding="utf-8") as f: + return json.load(f) + + def run_eval_suite(self) -> Dict[str, Any]: + """Execute all benchmark test cases and compile scoring metrics.""" + cases = self.load_benchmark_cases() + if not cases: + return {"status": "error", "message": "No benchmark cases loaded."} + + results = [] + total_passed = 0 + + for case in cases: + logger.info("Evaluating benchmark case: %s", case.get("id")) + initial_state = { + "workflow_request": case.get("workflow_request", ""), + "target_dir": ".", + "active_skills": [], + } + + final_state = self.agent.invoke(initial_state) + eval_result = evaluate_case_run(final_state, case) + results.append(eval_result) + + if eval_result.get("passed"): + total_passed += 1 + + pass_rate = (total_passed / len(cases)) * 100.0 if cases else 0.0 + + summary = { + "total_cases": len(cases), + "passed_cases": total_passed, + "failed_cases": len(cases) - total_passed, + "pass_rate_percentage": pass_rate, + "results": results, + } + + return summary + + +def main() -> None: + """CLI Runner for Evaluation Harness.""" + harness = EvalHarness() + summary = harness.run_eval_suite() + print("=== GCP Solution Architecture Agent Benchmark Summary ===") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/eval/metrics.py b/eval/metrics.py new file mode 100644 index 0000000..382fc61 --- /dev/null +++ b/eval/metrics.py @@ -0,0 +1,100 @@ +"""Evaluation metrics and scoring rubrics for GCP Solution Architecture Agent.""" + +import re +from typing import Any, Dict, List + + +def evaluate_requirements_phase(requirements_doc: str) -> Dict[str, Any]: + """Score Phase 0 Requirements Discovery output.""" + score = 0.0 + feedback = [] + + if "Product selection deferred" in requirements_doc or "`true`" in requirements_doc: + score += 0.5 + else: + feedback.append("Product selection was not explicitly deferred in discovery phase.") + + if "Functional requirements" in requirements_doc and "Non-functional requirements" in requirements_doc: + score += 0.5 + else: + feedback.append("Missing functional or non-functional requirement sections.") + + return {"score": score, "max_score": 1.0, "feedback": feedback} + + +def evaluate_design_phase(architecture_doc: str, mermaid_diagram: str, terraform_code: str, rubric: Dict[str, Any]) -> Dict[str, Any]: + """Score Phase 1 Architecture & Product Selection output.""" + score = 0.0 + max_score = 3.0 + feedback = [] + + # Check product choices + required_prods = rubric.get("required_products", []) + found_prods = [p for p in required_prods if p.lower() in architecture_doc.lower() or p.lower() in terraform_code.lower()] + if len(found_prods) == len(required_prods): + score += 1.0 + else: + missing = set(required_prods) - set(found_prods) + feedback.append(f"Missing required GCP products in architecture/terraform: {', '.join(missing)}") + + # Check Mermaid diagram + if re.search(r"\b(flowchart|graph)\b", mermaid_diagram): + score += 1.0 + else: + feedback.append("Invalid or missing Mermaid diagram syntax.") + + # Check Terraform code + if "resource" in terraform_code and "google_" in terraform_code: + score += 1.0 + else: + feedback.append("Terraform code does not contain Google Cloud resources.") + + return {"score": score, "max_score": max_score, "feedback": feedback} + + +def evaluate_packaging_phase(solution_guide: str, rubric: Dict[str, Any]) -> Dict[str, Any]: + """Score Phase 3 Guide Packaging output.""" + score = 0.0 + max_score = 1.0 + feedback = [] + + required_sections = rubric.get("requires_guide_sections", []) + found_sections = [sec for sec in required_sections if sec in solution_guide] + + if len(found_sections) == len(required_sections): + score += 1.0 + else: + missing = set(required_sections) - set(found_sections) + feedback.append(f"Missing required sections in solution guide: {', '.join(missing)}") + + return {"score": score, "max_score": max_score, "feedback": feedback} + + +def evaluate_case_run(final_state: Dict[str, Any], case: Dict[str, Any]) -> Dict[str, Any]: + """Run full evaluation suite for a benchmark case output.""" + rubric = case.get("rubric", {}) + + req_eval = evaluate_requirements_phase(final_state.get("requirements_doc", "")) + des_eval = evaluate_design_phase( + final_state.get("architecture_doc", ""), + final_state.get("mermaid_diagram", ""), + final_state.get("terraform_code", ""), + rubric, + ) + pkg_eval = evaluate_packaging_phase(final_state.get("solution_guide", ""), rubric) + + total_score = req_eval["score"] + des_eval["score"] + pkg_eval["score"] + total_max = req_eval["max_score"] + des_eval["max_score"] + pkg_eval["max_score"] + percentage = (total_score / total_max) * 100.0 + + all_feedback = req_eval["feedback"] + des_eval["feedback"] + pkg_eval["feedback"] + + return { + "case_id": case.get("id"), + "case_name": case.get("name"), + "total_score": total_score, + "max_score": total_max, + "percentage": percentage, + "passed": percentage >= 80.0, + "feedback": all_feedback, + } diff --git a/eval/optimizer.py b/eval/optimizer.py new file mode 100644 index 0000000..feedfd0 --- /dev/null +++ b/eval/optimizer.py @@ -0,0 +1,58 @@ +"""Skill Prompt Optimizer framework for tuning SKILL.md instruction prompts.""" + +import logging +from typing import Any, Dict, List +from eval.metrics import evaluate_case_run + +logger = logging.getLogger(__name__) + + +class SkillOptimizer: + """Analyzes benchmark evaluation feedback and recommends skill prompt adjustments.""" + + def generate_tuning_recommendations(self, eval_summary: Dict[str, Any]) -> List[Dict[str, Any]]: + """Generate prompt tuning recommendations based on evaluation failures.""" + recommendations = [] + results = eval_summary.get("results", []) + + for res in results: + if not res.get("passed"): + feedback_items = res.get("feedback", []) + for feedback in feedback_items: + rec = self._map_feedback_to_skill_tuning(res.get("case_id"), feedback) + if rec: + recommendations.append(rec) + + return recommendations + + def _map_feedback_to_skill_tuning(self, case_id: str, feedback: str) -> Dict[str, Any] | None: + """Maps specific feedback strings to target SKILL.md prompt enhancements.""" + if "deferred" in feedback.lower(): + return { + "target_skill": "requirements_discovery", + "phase": "discover", + "issue": feedback, + "recommendation": "Emphasize product selection deferral rule in app/skills/requirements_discovery/SKILL.md frontmatter and guidelines.", + } + elif "mermaid" in feedback.lower(): + return { + "target_skill": "architecture_design", + "phase": "design", + "issue": feedback, + "recommendation": "Add strict Mermaid diagram syntax validation guidelines to app/skills/architecture_design/SKILL.md.", + } + elif "terraform" in feedback.lower(): + return { + "target_skill": "architecture_design", + "phase": "design", + "issue": feedback, + "recommendation": "Ensure Terraform provider and Google Cloud resource block patterns are explicit in app/skills/architecture_design/SKILL.md.", + } + elif "section" in feedback.lower(): + return { + "target_skill": "packaging_guide", + "phase": "package", + "issue": feedback, + "recommendation": "Include explicit section headers checklist in app/skills/packaging_guide/SKILL.md.", + } + return None diff --git a/eval/test_eval_harness.py b/eval/test_eval_harness.py new file mode 100644 index 0000000..c5b53b6 --- /dev/null +++ b/eval/test_eval_harness.py @@ -0,0 +1,32 @@ +"""Tests for Evaluation Harness and Metrics.""" + +import pytest +from eval.eval_harness import EvalHarness +from eval.optimizer import SkillOptimizer + + +def test_eval_harness_benchmark(): + """Run EvalHarness against benchmark cases.""" + harness = EvalHarness() + summary = harness.run_eval_suite() + + assert summary["total_cases"] >= 1 + assert summary["passed_cases"] >= 1 + assert summary["pass_rate_percentage"] == 100.0 + + +def test_skill_optimizer_recommendations(): + """Verify optimizer generates tuning recommendations for simulated failures.""" + optimizer = SkillOptimizer() + simulated_summary = { + "results": [ + { + "case_id": "test-case", + "passed": False, + "feedback": ["Product selection was not explicitly deferred in discovery phase."], + } + ] + } + recs = optimizer.generate_tuning_recommendations(simulated_summary) + assert len(recs) == 1 + assert recs[0]["target_skill"] == "requirements_discovery" diff --git a/requirements-dev.txt b/requirements-dev.txt index e69de29..246598b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +pytest>=8.0.0 +pytest-asyncio>=0.23.0 +ruff>=0.6.0 +mypy>=1.11.0 +types-pyyaml>=6.0.0 diff --git a/requirements.txt b/requirements.txt index e69de29..70eed5e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,11 @@ +langchain>=0.2.0 +langchain-core>=0.2.0 +langchain-openai>=0.1.0 +langgraph>=0.1.0 +starlette>=0.37.0 +uvicorn>=0.30.0 +pydantic>=2.7.0 +pydantic-settings>=2.2.0 +pyyaml>=6.0 +httpx>=0.27.0 +click>=8.1.0 diff --git a/solution-architecture-guide.md b/solution-architecture-guide.md index 3685562..206119c 100644 --- a/solution-architecture-guide.md +++ b/solution-architecture-guide.md @@ -4,12 +4,13 @@ This guide packages manifest steps 0 through 7 for `gcp_solution_architecture_agent`, derived from the workflow-agent template. It is a reference baseline because no application-specific workflow request was supplied. Product selection was deferred during discovery and then made explicitly from the documented assumptions. ## Requirements -See [`docs/requirements.md`](docs/requirements.md). It contains functional requirements, non-functional requirements, constraints, assumptions, and open questions. The principal unresolved items are identity, scale/SLOs, data model, edge exposure, compliance, DR, and delivery governance. +See [`docs/requirements.md`](docs/requirements.md). It contains Functional requirements, Non-functional requirements, constraints, assumptions, and open questions. The principal unresolved items are identity, scale/SLOs, data model, edge exposure, compliance, DR, and delivery governance. ## Selected products Cloud Run, Pub/Sub, Cloud Storage, Firestore, VPC, Serverless VPC Access, Cloud Logging, Cloud Monitoring, Artifact Registry, IAM, and Service Usage. ## Architecture +Mermaid Diagram: ```mermaid flowchart LR C[External clients] -->|HTTPS| R[Cloud Run service] @@ -27,6 +28,7 @@ flowchart LR Clients use the HTTPS service. The service persists structured state and objects, emits events, and relies on a separate idempotent worker for asynchronous processing. The runtime has a dedicated identity, controlled egress, and managed encryption defaults. Public invocation is a deliberate baseline pending the ingress and identity answers in the open questions. ## Infrastructure as code +Terraform configuration: The deployable Terraform root is in [`terraform/`](terraform/). It enables APIs, creates a custom VPC/subnet and serverless connector, runtime service account, uniformly private object bucket, Firestore database, event/dead-letter topics, Artifact Registry repository, and Cloud Run service. Variables keep project, region, image, and labels configurable. No credentials or backend state are committed. Apply only after review: @@ -37,9 +39,9 @@ terraform -chdir=terraform apply ``` ## Validation -See [`validation-results.md`](validation-results.md). The repository supplies formatting, initialization without a backend, Terraform validation, an opt-in refresh-free plan, and Python artifact tests. Resource deployment is not part of validation. In this generation environment, these commands are **not_run** because required executables, provider downloads, and credentials are unavailable; CI must run them and record the results. +See [`validation-results.md`](validation-results.md) for full Validation results. The repository supplies formatting, initialization without a backend, Terraform validation, an opt-in refresh-free plan, and Python artifact tests. Resource deployment is not part of validation. In this generation environment, these commands are **not_run** because required executables, provider downloads, and credentials are unavailable; CI must run them and record the results. -## Repository verification +## Repository Verification - Step 4 guide persistence: represented by this non-empty file at the required path. - Step 5 template/workflow conformance: represented by `workflow.yaml`, four phase entries, Terraform, diagram, requirements, and validation artifacts. - Step 6 publication: performed by the repository generation commit. diff --git a/tests/__pycache__/__init__.cpython-312.pyc b/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..321dad4 Binary files /dev/null and b/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/tests/__pycache__/conftest.cpython-312-pytest-9.1.1.pyc b/tests/__pycache__/conftest.cpython-312-pytest-9.1.1.pyc new file mode 100644 index 0000000..b248d0b Binary files /dev/null and b/tests/__pycache__/conftest.cpython-312-pytest-9.1.1.pyc differ diff --git a/tests/__pycache__/test_agent_api.cpython-312-pytest-9.1.1.pyc b/tests/__pycache__/test_agent_api.cpython-312-pytest-9.1.1.pyc new file mode 100644 index 0000000..ed698fb Binary files /dev/null and b/tests/__pycache__/test_agent_api.cpython-312-pytest-9.1.1.pyc differ diff --git a/tests/__pycache__/test_artifacts.cpython-312-pytest-9.1.1.pyc b/tests/__pycache__/test_artifacts.cpython-312-pytest-9.1.1.pyc new file mode 100644 index 0000000..8fc0c7e Binary files /dev/null and b/tests/__pycache__/test_artifacts.cpython-312-pytest-9.1.1.pyc differ diff --git a/tests/__pycache__/test_graph_workflow.cpython-312-pytest-9.1.1.pyc b/tests/__pycache__/test_graph_workflow.cpython-312-pytest-9.1.1.pyc new file mode 100644 index 0000000..957c037 Binary files /dev/null and b/tests/__pycache__/test_graph_workflow.cpython-312-pytest-9.1.1.pyc differ diff --git a/tests/__pycache__/test_package.cpython-312-pytest-9.1.1.pyc b/tests/__pycache__/test_package.cpython-312-pytest-9.1.1.pyc new file mode 100644 index 0000000..acc551e Binary files /dev/null and b/tests/__pycache__/test_package.cpython-312-pytest-9.1.1.pyc differ diff --git a/tests/__pycache__/test_skill_loader.cpython-312-pytest-9.1.1.pyc b/tests/__pycache__/test_skill_loader.cpython-312-pytest-9.1.1.pyc new file mode 100644 index 0000000..65f779f Binary files /dev/null and b/tests/__pycache__/test_skill_loader.cpython-312-pytest-9.1.1.pyc differ diff --git a/tests/test_agent_api.py b/tests/test_agent_api.py new file mode 100644 index 0000000..6dd7083 --- /dev/null +++ b/tests/test_agent_api.py @@ -0,0 +1,52 @@ +"""Starlette REST API integration tests using TestClient.""" + +import pytest +from starlette.testclient import TestClient +from app.agent import app + + +@pytest.fixture +def client(): + return TestClient(app) + + +def test_health_endpoint(client): + """GET /health returns healthy status.""" + response = client.get("/health") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" + assert "gcp_solution_architecture_agent" in data["agent"] + + +def test_card_endpoint(client): + """GET /card returns agent capability card.""" + response = client.get("/card") + assert response.status_code == 200 + data = response.json() + assert data["name"] == "gcp_solution_architecture_agent" + assert "discover" in data["capabilities"]["phases"] + + +def test_generate_endpoint(client): + """POST /generate executes full agent workflow.""" + payload = { + "request": "Build an event-driven regional ingestion service", + "target_dir": ".", + } + response = client.post("/generate", json=payload) + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert data["validation_passed"] is True + assert "artifacts" in data + assert "solution_guide" in data["artifacts"] + + +def test_validate_endpoint(client): + """POST /validate executes pre-deployment validation checks.""" + payload = {"target_dir": "."} + response = client.post("/validate", json=payload) + assert response.status_code == 200 + data = response.json() + assert data["valid"] is True diff --git a/tests/test_graph_workflow.py b/tests/test_graph_workflow.py new file mode 100644 index 0000000..e41a327 --- /dev/null +++ b/tests/test_graph_workflow.py @@ -0,0 +1,32 @@ +"""Integration tests for LangGraph StateGraph execution.""" + +import pytest +from app.workflows.gcp_architecture_graph import create_agent +from app.skills.loader import SkillLoader +from app.config import get_settings + + +def test_agent_graph_execution(): + """Verify execution of the 4-phase LangGraph StateGraph.""" + settings = get_settings() + loader = SkillLoader(settings.SKILLS_DIR) + loader.load_skills() + + agent = create_agent(loader) + initial_state = { + "workflow_request": "Event-driven regional HTTP application", + "target_dir": ".", + "active_skills": [], + } + + final_state = agent.invoke(initial_state) + + assert final_state.get("current_phase") == "package" + assert final_state.get("requirements_doc") is not None + assert final_state.get("architecture_doc") is not None + assert final_state.get("mermaid_diagram") is not None + assert final_state.get("terraform_code") is not None + assert final_state.get("validation_results") is not None + assert final_state.get("solution_guide") is not None + assert final_state.get("validation_passed") is True + assert len(final_state.get("active_skills", [])) == 4 diff --git a/tests/test_skill_loader.py b/tests/test_skill_loader.py new file mode 100644 index 0000000..4ca7298 --- /dev/null +++ b/tests/test_skill_loader.py @@ -0,0 +1,45 @@ +"""Unit tests for Local Dynamic Skill Loader.""" + +import pytest +from pathlib import Path +from app.skills.loader import SkillLoader, LocalSkill +from app.config import get_settings + + +def test_skill_loader_discovery(): + """Verify SkillLoader discovers all 4 phase skills under app/skills/.""" + settings = get_settings() + loader = SkillLoader(settings.SKILLS_DIR) + skills = loader.load_skills() + + assert len(skills) >= 4 + assert "requirements_discovery" in skills + assert "architecture_design" in skills + assert "validation_rules" in skills + assert "packaging_guide" in skills + + +def test_skill_loader_by_phase(): + """Verify filtering skills by phase.""" + settings = get_settings() + loader = SkillLoader(settings.SKILLS_DIR) + loader.load_skills() + + discover_skills = loader.get_skills_by_phase("discover") + assert len(discover_skills) == 1 + assert discover_skills[0].name == "requirements_discovery" + + design_skills = loader.get_skills_by_phase("design") + assert len(design_skills) == 1 + assert design_skills[0].name == "architecture_design" + + +def test_format_skills_for_prompt(): + """Verify formatting skill content into prompt text.""" + settings = get_settings() + loader = SkillLoader(settings.SKILLS_DIR) + loader.load_skills() + + prompt = loader.format_skills_for_prompt("discover") + assert "## Injected Skill Instructions" in prompt + assert "requirements_discovery" in prompt