This commit is contained in:
@@ -1,9 +1 @@
|
||||
"""Nodes package."""
|
||||
|
||||
from .design_node import design_node
|
||||
from .discover_node import discover_node
|
||||
from .package_node import package_node
|
||||
from .source_discover_node import source_discover_node
|
||||
from .validate_node import validate_node
|
||||
|
||||
__all__ = ["source_discover_node", "discover_node", "design_node", "validate_node", "package_node"]
|
||||
"""The nodes of the Execution graph."""
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
"""Phase 1: Architecture Design Node."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
from app.states.state import GCPArchitectureState
|
||||
from app.skills.loader import SkillLoader
|
||||
from app.tools.mcp_developer_knowledge import get_mcp_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def design_node(state: GCPArchitectureState, skill_loader: SkillLoader) -> Dict[str, Any]:
|
||||
"""Processes Phase 1 Design: select GCP managed products, generate Mermaid graph, and Terraform IaC."""
|
||||
logger.info("Executing design_node")
|
||||
skill_prompt = skill_loader.format_skills_for_prompt("design")
|
||||
|
||||
# Developer Knowledge MCP Grounding Search
|
||||
mcp_client = get_mcp_client()
|
||||
mcp_knowledge = mcp_client.search_documents(query="Cloud Run PubSub event-driven architecture", category="compute")
|
||||
docs = mcp_knowledge.get("documents", [])
|
||||
citations_str = "\n".join([f"- [{doc['title']}]({doc['uri']})" for doc in docs[:3]]) if docs else "- [Google Cloud Architecture Framework](https://cloud.google.com/architecture/framework)"
|
||||
|
||||
architecture_doc = f"""# Phase 1 — Architecture & Product Selection
|
||||
|
||||
## Selected Products
|
||||
- **Compute / Serving**: Google Cloud Run (Fully Managed Container Ingress & Stateless Execution)
|
||||
- **Messaging & Eventing**: Google Cloud Pub/Sub (Regional Event Bus for Asynchronous Decoupling)
|
||||
- **State & Storage**: Google Cloud Storage & Firestore (Database & Bucket Storage for Durable Audit Event Replay)
|
||||
- **Security & Identity**: Cloud IAM (Least Privilege Service Accounts) & KMS (Customer-Managed Encryption Keys)
|
||||
- **Artifact Registry**: Google Artifact Registry (OCI Container Image Hosting)
|
||||
|
||||
## Component Responsibilities
|
||||
1. **Cloud Run Service**: Accepts HTTPS requests, validates client signatures, enqueues events to Pub/Sub, returns 202 Accepted.
|
||||
2. **Pub/Sub Topic & Subscription**: Buffer incoming payloads, deliver events asynchronously with exponential backoff retries to consumer handlers.
|
||||
3. **Audit Bucket (GCS)**: Raw event retention for replay, payload audit, and operational troubleshooting.
|
||||
|
||||
## Security & Compliance
|
||||
- HTTPS ingress with TLS 1.3 encryption in transit.
|
||||
- Default Google-managed encryption at rest for Cloud Storage and Pub/Sub.
|
||||
- Cloud Run service account bound strictly to `roles/pubsub.publisher` and `roles/storage.objectCreator`.
|
||||
|
||||
## Grounded Documentation Citations (Google Developer Knowledge MCP)
|
||||
{citations_str}
|
||||
"""
|
||||
|
||||
mermaid_diagram = """flowchart TD
|
||||
Client[External HTTPS Client] -->|HTTPS POST /events| CloudRun[Google Cloud Run Service]
|
||||
CloudRun -->|Publish Event| PubSubTopic[Cloud Pub/Sub Topic]
|
||||
CloudRun -->|Write Raw Payload| GCSAudit[Cloud Storage Audit Bucket]
|
||||
PubSubTopic -->|Push Delivery| EventConsumer[Cloud Run Consumer Service]
|
||||
EventConsumer -->|Acknowledge| PubSubTopic
|
||||
"""
|
||||
|
||||
terraform_code = """# Google Cloud Solution Architecture Baseline
|
||||
terraform {
|
||||
required_version = ">= 1.5.0"
|
||||
required_providers {
|
||||
google = {
|
||||
source = "hashicorp/google"
|
||||
version = "~> 5.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "google" {
|
||||
project = var.project_id
|
||||
region = var.region
|
||||
}
|
||||
|
||||
# Cloud Run v2 Service
|
||||
resource "google_cloud_run_v2_service" "app_service" {
|
||||
name = "${var.environment}-app-service"
|
||||
location = var.region
|
||||
|
||||
template {
|
||||
containers {
|
||||
image = var.container_image
|
||||
ports {
|
||||
container_port = 8080
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Pub/Sub Topic for Event Ingestion
|
||||
resource "google_pubsub_topic" "event_ingestion" {
|
||||
name = "${var.environment}-event-ingestion-topic"
|
||||
labels = {
|
||||
environment = var.environment
|
||||
managed_by = "terraform"
|
||||
}
|
||||
}
|
||||
|
||||
# Cloud Storage Bucket for Event Replay Audit
|
||||
resource "google_storage_bucket" "audit_bucket" {
|
||||
name = "${var.project_id}-${var.environment}-audit-bucket"
|
||||
location = var.region
|
||||
force_destroy = false
|
||||
uniform_bucket_level_access = true
|
||||
|
||||
versioning {
|
||||
enabled = true
|
||||
}
|
||||
|
||||
lifecycle_rule {
|
||||
condition {
|
||||
age = 30
|
||||
}
|
||||
action {
|
||||
type = "Delete"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Least-Privilege IAM Service Account
|
||||
resource "google_service_account" "ingress_sa" {
|
||||
account_id = "${var.environment}-ingress-sa"
|
||||
display_name = "Cloud Run Ingress Identity"
|
||||
}
|
||||
|
||||
resource "google_pubsub_topic_iam_member" "publisher_binding" {
|
||||
topic = google_pubsub_topic.event_ingestion.name
|
||||
role = "roles/pubsub.publisher"
|
||||
member = "serviceAccount:${google_service_account.ingress_sa.email}"
|
||||
}
|
||||
"""
|
||||
|
||||
active_skills = state.get("active_skills", [])
|
||||
if "architecture_design" not in active_skills:
|
||||
active_skills.append("architecture_design")
|
||||
|
||||
return {
|
||||
"architecture_doc": architecture_doc,
|
||||
"mermaid_diagram": mermaid_diagram,
|
||||
"terraform_code": terraform_code,
|
||||
"product_selection_deferred": False,
|
||||
"current_phase": "design",
|
||||
"active_skills": active_skills,
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
"""Phase 0: Requirements Discovery Node."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
from app.states.state import GCPArchitectureState
|
||||
from app.skills.loader import SkillLoader
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def discover_node(state: GCPArchitectureState, skill_loader: SkillLoader) -> Dict[str, Any]:
|
||||
"""Processes Phase 0 Discovery: document functional/non-functional requirements with product selection deferred."""
|
||||
logger.info("Executing discover_node")
|
||||
skill_prompt = skill_loader.format_skills_for_prompt("discover")
|
||||
|
||||
# Generate or format requirements document baseline
|
||||
request_summary = state.get("workflow_request", "Event-driven HTTP application reference architecture")
|
||||
|
||||
requirements_doc = f"""# Step 0 — Requirements discovery
|
||||
|
||||
## Workflow request
|
||||
{request_summary}
|
||||
|
||||
## Functional requirements
|
||||
- Accept authenticated HTTPS requests from external clients.
|
||||
- Execute stateless application logic behind a versioned service endpoint.
|
||||
- Publish asynchronous domain events from the application.
|
||||
- Process events independently and tolerate retry/redelivery.
|
||||
- Persist durable objects and application state separately.
|
||||
- Expose operational logs, metrics, and audit-relevant events.
|
||||
- Support repeatable infrastructure changes through declarative IaC.
|
||||
|
||||
## Non-functional requirements
|
||||
- High availability within a selected Google Cloud region.
|
||||
- Horizontal scale for bursty HTTP traffic and asynchronous work.
|
||||
- At-least-once event delivery with idempotent consumers.
|
||||
- Encryption in transit and at rest using managed defaults initially.
|
||||
- Least-privilege runtime identities and private network egress where practical.
|
||||
- Observable deployments with structured logs and actionable health signals.
|
||||
- Reproducible, reviewable, non-deployment validation in CI.
|
||||
|
||||
## Constraints
|
||||
- Google Cloud is the target cloud; exact products are not selected in discovery.
|
||||
- Terraform must be deployable without embedding secrets or credentials.
|
||||
- The baseline must not provision resources during validation.
|
||||
- A container image must be supplied by the application delivery pipeline.
|
||||
- State backends, DNS ownership, identity federation, and organization policies are external concerns.
|
||||
|
||||
## Assumptions
|
||||
- A single region is acceptable for the initial deployment.
|
||||
- The application can be packaged as an OCI container listening on port 8080.
|
||||
- Events can use at-least-once semantics and consumers can deduplicate.
|
||||
- A dedicated Google Cloud project is available.
|
||||
- Managed encryption keys and public ingress are acceptable defaults pending review.
|
||||
|
||||
## Open questions
|
||||
- What are the actual API, event, data-retention, and compliance requirements?
|
||||
- Which clients and identity provider must authenticate requests?
|
||||
- What are traffic, payload-size, latency, RTO, and RPO targets?
|
||||
- Which data is relational, document, object, or analytical?
|
||||
|
||||
**Product selection deferred:** `true` for this phase.
|
||||
"""
|
||||
|
||||
active_skills = state.get("active_skills", [])
|
||||
if "requirements_discovery" not in active_skills:
|
||||
active_skills.append("requirements_discovery")
|
||||
|
||||
return {
|
||||
"requirements_doc": requirements_doc,
|
||||
"product_selection_deferred": True,
|
||||
"current_phase": "discover",
|
||||
"active_skills": active_skills,
|
||||
}
|
||||
560
app/nodes/execution_nodes.py
Normal file
560
app/nodes/execution_nodes.py
Normal file
@@ -0,0 +1,560 @@
|
||||
"""The nodes of the Execution graph (ADR-0003); app/workflows/execution_graph.py wires them.
|
||||
|
||||
Only orchestrator_turn and await_user change the Ledger; parallel Specialist tasks
|
||||
append outcomes that the next Orchestrator turn applies in order.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from langgraph.types import Command, Send, interrupt
|
||||
|
||||
from app.adk.runtime import (
|
||||
AgentRuntime,
|
||||
AgentTask,
|
||||
BudgetExhausted,
|
||||
NeedsInput,
|
||||
Outcome,
|
||||
RoleSpec,
|
||||
Stopped,
|
||||
Submitted,
|
||||
ToolFactory,
|
||||
ToolSpec,
|
||||
TurnEnded,
|
||||
)
|
||||
from app.recorder import ExecutionRecorder
|
||||
from app.states.execution_state import ExecutionState
|
||||
from app.states.ledger import Ledger, LedgerRefusal
|
||||
from app.tools.execution import ORCHESTRATOR, OrchestratorToolbox, specialist_tools
|
||||
from app.workflows.deliverables import (
|
||||
DEPENDENCY_GRAPH,
|
||||
PHASE_ORDER,
|
||||
DeliverableKind,
|
||||
inputs_of,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NO_USER_REPLY = (
|
||||
"No user is available. State your assumption, record it in the Deliverable under "
|
||||
"'Assumptions', and proceed."
|
||||
)
|
||||
GUIDE_FILENAME = "solution-architecture-guide.md"
|
||||
|
||||
|
||||
def _no_tools(role: RoleSpec, deliverable: str | None) -> Sequence[ToolSpec]:
|
||||
return ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExecutionNodes:
|
||||
"""The node functions of one Execution graph, bound to its runtime and recorder."""
|
||||
|
||||
orchestrator_turn: Callable[[ExecutionState], Awaitable[Command]]
|
||||
# Receives its Send payload, not the Execution state.
|
||||
specialist_task: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]
|
||||
await_user: Callable[[ExecutionState], Awaitable[Command]]
|
||||
finalize: Callable[[ExecutionState], Awaitable[dict[str, Any]]]
|
||||
end_with_warnings: Callable[[ExecutionState], Awaitable[dict[str, Any]]]
|
||||
|
||||
|
||||
def make_execution_nodes(
|
||||
*,
|
||||
runtime: AgentRuntime,
|
||||
roles: Mapping[str, RoleSpec],
|
||||
recorder: ExecutionRecorder,
|
||||
output_root: Path,
|
||||
max_orchestrator_turns: int = 100,
|
||||
tool_factory: ToolFactory | None = None,
|
||||
) -> ExecutionNodes:
|
||||
tool_factory = tool_factory or _no_tools
|
||||
|
||||
async def record_progress(
|
||||
state: ExecutionState, ledger: Ledger, status: str | None = None
|
||||
) -> int:
|
||||
recorded = state.get("events_recorded", 0)
|
||||
for event in ledger.events[recorded:]:
|
||||
logger.info(
|
||||
"[%s] ledger %s %s by %s%s",
|
||||
state["execution_id"],
|
||||
event.type,
|
||||
event.subject,
|
||||
event.actor,
|
||||
f" ({event.detail})" if event.detail else "",
|
||||
)
|
||||
await recorder.record_events(state["execution_id"], ledger.events[recorded:])
|
||||
if status:
|
||||
await recorder.record_status(
|
||||
state["execution_id"],
|
||||
session_id=state["session_id"],
|
||||
workflow_request=state["workflow_request"],
|
||||
status=status,
|
||||
current_phase=_current_phase(ledger),
|
||||
details={
|
||||
"revisions": sum(
|
||||
ledger.revisions(kind) for kind in DEPENDENCY_GRAPH
|
||||
),
|
||||
"warnings": state.get("warnings", []),
|
||||
},
|
||||
)
|
||||
return len(ledger.events)
|
||||
|
||||
async def orchestrator_turn(state: ExecutionState) -> Command:
|
||||
ledger = Ledger.from_dict(state["ledger"])
|
||||
deliverables = dict(state.get("deliverables", {}))
|
||||
pending_questions = list(state.get("pending_questions", []))
|
||||
notes = list(state.get("notes", []))
|
||||
pending_permission = state.get("pending_permission")
|
||||
warnings = list(state.get("warnings", []))
|
||||
|
||||
outcomes = state.get("outcomes", [])
|
||||
for record in outcomes[state.get("outcomes_applied", 0) :]:
|
||||
kind = DeliverableKind(record["kind"])
|
||||
try:
|
||||
if record["type"] == "submitted":
|
||||
ledger.submit(kind, record["attempt"], actor=record["role"])
|
||||
deliverables[kind.value] = {
|
||||
"content": record["content"],
|
||||
"attachments": record["attachments"],
|
||||
}
|
||||
await recorder.record_submission(
|
||||
state["execution_id"],
|
||||
kind.value,
|
||||
record["content"],
|
||||
record["attachments"],
|
||||
)
|
||||
continue
|
||||
ledger.release(kind, record["attempt"], reason=record["type"])
|
||||
except LedgerRefusal as exc:
|
||||
notes.append(f"Discarded a superseded result for {kind}: {exc}")
|
||||
continue
|
||||
if record["type"] == "needs_input" and record.get("permission"):
|
||||
if state.get("allow_cloud_read") is None:
|
||||
requested = (pending_permission or {}).get("requested_by", [])
|
||||
pending_permission = {
|
||||
"tier": record["permission"],
|
||||
"requested_by": [*requested, kind.value],
|
||||
}
|
||||
continue
|
||||
questions = record.get("questions") or [
|
||||
(
|
||||
f"The {record['role']} stopped: {record['summary']} "
|
||||
"How should it proceed?"
|
||||
)
|
||||
]
|
||||
pending_questions.append({"kind": kind.value, "questions": questions})
|
||||
|
||||
turns = state.get("turns", 0) + 1
|
||||
update: dict[str, Any] = {
|
||||
"deliverables": deliverables,
|
||||
"pending_questions": pending_questions,
|
||||
"pending_permission": pending_permission,
|
||||
"outcomes_applied": len(outcomes),
|
||||
"turns": turns,
|
||||
}
|
||||
|
||||
def finish_with(goto: str, **extra: Any) -> Command:
|
||||
return Command(
|
||||
update={**update, "ledger": ledger.to_dict(), "notes": notes, **extra},
|
||||
goto=goto,
|
||||
)
|
||||
|
||||
if pending_permission and not state.get("pre_approved"):
|
||||
update["events_recorded"] = await record_progress(
|
||||
state, ledger, "awaiting_user"
|
||||
)
|
||||
return finish_with("await_user", prompt={"permission": pending_permission})
|
||||
|
||||
if turns > max_orchestrator_turns:
|
||||
warnings.append(
|
||||
f"Stopped after {max_orchestrator_turns} Orchestrator turns without "
|
||||
"finishing."
|
||||
)
|
||||
return finish_with("end_with_warnings", warnings=warnings)
|
||||
|
||||
toolbox = OrchestratorToolbox(ledger, deliverables, pending_questions)
|
||||
task = AgentTask(
|
||||
role=ORCHESTRATOR,
|
||||
prompt="Decide the next step for this Execution using your tools.",
|
||||
context=_orchestrator_context(state, ledger, pending_questions, notes),
|
||||
)
|
||||
started = time.monotonic()
|
||||
outcome = await runtime.run(roles[ORCHESTRATOR], task, toolbox.tools())
|
||||
logger.info(
|
||||
"[%s] orchestrator turn %d (%s phase) ended in %.0fs: %s",
|
||||
state["execution_id"],
|
||||
turns,
|
||||
_current_phase(ledger),
|
||||
time.monotonic() - started,
|
||||
_summarise_outcome(outcome),
|
||||
)
|
||||
notes = []
|
||||
pre_approved = bool(state.get("pre_approved"))
|
||||
conversation = list(state.get("conversation", []))
|
||||
|
||||
def ask(prompt: dict[str, Any], **extra: Any) -> Command:
|
||||
conversation.append({"from": ORCHESTRATOR, "text": _render_prompt(prompt)})
|
||||
return finish_with(
|
||||
"await_user", prompt=prompt, conversation=conversation, **extra
|
||||
)
|
||||
|
||||
if isinstance(outcome, TurnEnded) and outcome.tool == "delegate":
|
||||
sends = [
|
||||
Send(
|
||||
"specialist_task",
|
||||
_specialist_input(state, ledger, deliverables, kind, attempt),
|
||||
)
|
||||
for kind, attempt in toolbox.started.items()
|
||||
]
|
||||
update["events_recorded"] = await record_progress(state, ledger)
|
||||
return Command(
|
||||
update={**update, "ledger": ledger.to_dict(), "notes": notes},
|
||||
goto=sends,
|
||||
)
|
||||
|
||||
if isinstance(outcome, TurnEnded) and outcome.tool == "finish":
|
||||
update["events_recorded"] = await record_progress(state, ledger)
|
||||
return finish_with("finalize")
|
||||
|
||||
if isinstance(outcome, TurnEnded) and outcome.tool == "escalate":
|
||||
warnings.append(toolbox.escalation or "Escalated without detail.")
|
||||
update["events_recorded"] = await record_progress(state, ledger)
|
||||
return finish_with("end_with_warnings", warnings=warnings)
|
||||
|
||||
if isinstance(outcome, TurnEnded) and outcome.tool == "request_approval":
|
||||
update["events_recorded"] = await record_progress(
|
||||
state, ledger, None if pre_approved else "awaiting_user"
|
||||
)
|
||||
return ask({"approval": toolbox.approval})
|
||||
|
||||
if isinstance(outcome, TurnEnded) and outcome.tool == "ask_user":
|
||||
update["events_recorded"] = await record_progress(
|
||||
state, ledger, None if pre_approved else "awaiting_user"
|
||||
)
|
||||
return ask(
|
||||
{"questions": toolbox.questions},
|
||||
pending_questions=[],
|
||||
awaiting_questions=pending_questions,
|
||||
)
|
||||
|
||||
message = _describe_unfinished(outcome)
|
||||
if pre_approved:
|
||||
warnings.append(f"The Orchestrator could not continue: {message}")
|
||||
update["events_recorded"] = await record_progress(state, ledger)
|
||||
return finish_with("end_with_warnings", warnings=warnings)
|
||||
update["events_recorded"] = await record_progress(
|
||||
state, ledger, "awaiting_user"
|
||||
)
|
||||
return ask({"questions": [{"from": ORCHESTRATOR, "question": message}]})
|
||||
|
||||
async def specialist_task(task_input: dict[str, Any]) -> dict[str, Any]:
|
||||
role = roles[task_input["role"]]
|
||||
kind = DeliverableKind(task_input["kind"])
|
||||
tools = specialist_tools(
|
||||
kind,
|
||||
extra=tool_factory(role, kind.value),
|
||||
allow_cloud_read=task_input["allow_cloud_read"],
|
||||
)
|
||||
task = AgentTask(
|
||||
role=role.name,
|
||||
prompt=f"Produce the {kind} Deliverable.",
|
||||
context=task_input["context"],
|
||||
deliverable=kind.value,
|
||||
)
|
||||
logger.info(
|
||||
"[%s] %s started %s (attempt %d)",
|
||||
task_input["execution_id"],
|
||||
role.name,
|
||||
kind,
|
||||
task_input["attempt"],
|
||||
)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
outcome: Outcome = await runtime.run(role, task, tools)
|
||||
except Exception as exc:
|
||||
logger.exception("Specialist %s failed on %s", role.name, kind)
|
||||
outcome = Stopped(f"it failed with {type(exc).__name__}: {exc}")
|
||||
logger.info(
|
||||
"[%s] %s finished %s in %.0fs: %s",
|
||||
task_input["execution_id"],
|
||||
role.name,
|
||||
kind,
|
||||
time.monotonic() - started,
|
||||
_summarise_outcome(outcome),
|
||||
)
|
||||
return {
|
||||
"outcomes": [
|
||||
_outcome_record(role.name, kind, task_input["attempt"], outcome)
|
||||
]
|
||||
}
|
||||
|
||||
async def await_user(state: ExecutionState) -> Command:
|
||||
prompt = state["prompt"] or {}
|
||||
if state.get("pre_approved"):
|
||||
reply = _automatic_reply(prompt, state.get("scripted_answers", []))
|
||||
else:
|
||||
reply = interrupt(prompt)
|
||||
|
||||
ledger = Ledger.from_dict(state["ledger"])
|
||||
conversation = list(state.get("conversation", []))
|
||||
update: dict[str, Any] = {"prompt": None}
|
||||
|
||||
if "approval" in prompt:
|
||||
phase = prompt["approval"]["phase"]
|
||||
decision = reply.get("approval") if isinstance(reply, Mapping) else None
|
||||
if (
|
||||
isinstance(decision, Mapping)
|
||||
and decision.get("phase") == phase
|
||||
and decision.get("decision") in {"approve", "changes"}
|
||||
):
|
||||
phase_id = next(p for p in PHASE_ORDER if p.value == phase)
|
||||
if decision["decision"] == "approve":
|
||||
ledger.approve(phase_id, actor="user")
|
||||
text = f"Approved {phase}."
|
||||
else:
|
||||
feedback = str(decision.get("feedback", ""))
|
||||
ledger.request_changes(phase_id, feedback, actor="user")
|
||||
text = f"Requested changes to {phase}: {feedback}"
|
||||
conversation.append({"from": "user", "text": text})
|
||||
else:
|
||||
conversation.append({"from": "user", "text": _reply_text(reply)})
|
||||
elif "permission" in prompt:
|
||||
tier = prompt["permission"]["tier"]
|
||||
permission = reply.get("permission") if isinstance(reply, Mapping) else None
|
||||
grant = permission.get(tier) if isinstance(permission, Mapping) else None
|
||||
if isinstance(grant, bool):
|
||||
update["allow_cloud_read"] = grant
|
||||
update["pending_permission"] = None
|
||||
verdict = "granted" if grant else "denied"
|
||||
conversation.append(
|
||||
{
|
||||
"from": "user",
|
||||
"text": f"Permission to use {tier} tools {verdict}.",
|
||||
}
|
||||
)
|
||||
else:
|
||||
conversation.append({"from": "user", "text": _reply_text(reply)})
|
||||
else:
|
||||
text = _reply_text(reply)
|
||||
conversation.append({"from": "user", "text": text})
|
||||
answers = {
|
||||
kind: list(items) for kind, items in state.get("answers", {}).items()
|
||||
}
|
||||
for asked in state.get("awaiting_questions", []):
|
||||
answers.setdefault(asked["kind"], []).append(
|
||||
{"questions": list(asked["questions"]), "reply": text}
|
||||
)
|
||||
update.update(answers=answers, awaiting_questions=[])
|
||||
|
||||
update["events_recorded"] = await record_progress(state, ledger)
|
||||
update.update(ledger=ledger.to_dict(), conversation=conversation)
|
||||
return Command(update=update, goto="orchestrator_turn")
|
||||
|
||||
async def finalize(state: ExecutionState) -> dict[str, Any]:
|
||||
ledger = Ledger.from_dict(state["ledger"])
|
||||
_write_outputs(
|
||||
output_root / "deliverables" / "executions" / state["execution_id"],
|
||||
state.get("deliverables", {}),
|
||||
)
|
||||
await record_progress(state, ledger, "completed")
|
||||
return {"status": "completed"}
|
||||
|
||||
async def end_with_warnings(state: ExecutionState) -> dict[str, Any]:
|
||||
ledger = Ledger.from_dict(state["ledger"])
|
||||
await record_progress(state, ledger, "completed_with_warnings")
|
||||
return {"status": "completed_with_warnings"}
|
||||
|
||||
return ExecutionNodes(
|
||||
orchestrator_turn=orchestrator_turn,
|
||||
specialist_task=specialist_task,
|
||||
await_user=await_user,
|
||||
finalize=finalize,
|
||||
end_with_warnings=end_with_warnings,
|
||||
)
|
||||
|
||||
|
||||
# --- helpers ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _summarise_outcome(outcome: Outcome) -> str:
|
||||
if isinstance(outcome, TurnEnded):
|
||||
return f"called {outcome.tool}"
|
||||
if isinstance(outcome, Submitted):
|
||||
return f"submitted ({len(outcome.content)} chars)"
|
||||
if isinstance(outcome, NeedsInput):
|
||||
return f"needs input ({len(outcome.questions)} questions)"
|
||||
if isinstance(outcome, BudgetExhausted):
|
||||
return f"budget exhausted: {outcome.summary[:200]}"
|
||||
return f"stopped: {outcome.message[:200]}"
|
||||
|
||||
|
||||
def _current_phase(ledger: Ledger) -> str:
|
||||
for phase in PHASE_ORDER:
|
||||
if ledger.phase_status(phase).value != "approved":
|
||||
return phase.value
|
||||
return PHASE_ORDER[-1].value
|
||||
|
||||
|
||||
def _orchestrator_context(
|
||||
state: ExecutionState,
|
||||
ledger: Ledger,
|
||||
pending_questions: Sequence[Mapping[str, Any]],
|
||||
notes: Sequence[str],
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"workflow_request": state["workflow_request"],
|
||||
"pre_approved": bool(state.get("pre_approved")),
|
||||
"allow_cloud_read": state.get("allow_cloud_read"),
|
||||
"deliverables": {
|
||||
kind.value: {
|
||||
"phase": spec.phase.value,
|
||||
"status": ledger.status(kind).value,
|
||||
"revisions": ledger.revisions(kind),
|
||||
"feedback": list(ledger.feedback(kind)),
|
||||
}
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
},
|
||||
"phases": {
|
||||
phase.value: ledger.phase_status(phase).value for phase in PHASE_ORDER
|
||||
},
|
||||
"ready_work": [kind.value for kind in ledger.ready_work()],
|
||||
"phases_ready_for_approval": [
|
||||
p.value for p in ledger.phases_ready_for_approval()
|
||||
],
|
||||
"pending_questions": list(pending_questions),
|
||||
"conversation": list(state.get("conversation", []))[-20:],
|
||||
"notes": list(notes),
|
||||
}
|
||||
|
||||
|
||||
def _specialist_input(
|
||||
state: ExecutionState,
|
||||
ledger: Ledger,
|
||||
deliverables: Mapping[str, Mapping[str, Any]],
|
||||
kind: DeliverableKind,
|
||||
attempt: int,
|
||||
) -> dict[str, Any]:
|
||||
spec = DEPENDENCY_GRAPH[kind]
|
||||
return {
|
||||
"execution_id": state["execution_id"],
|
||||
"role": spec.specialist,
|
||||
"kind": kind.value,
|
||||
"attempt": attempt,
|
||||
"allow_cloud_read": state.get("allow_cloud_read"),
|
||||
"context": {
|
||||
"workflow_request": state["workflow_request"],
|
||||
"pre_approved": bool(state.get("pre_approved")),
|
||||
"dependencies": {
|
||||
dependency.value: dict(deliverables[dependency.value])
|
||||
for dependency in inputs_of(kind)
|
||||
},
|
||||
"feedback": list(ledger.feedback(kind)),
|
||||
"answers": list(state.get("answers", {}).get(kind.value, [])),
|
||||
"attempt": attempt,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _outcome_record(
|
||||
role: str, kind: DeliverableKind, attempt: int, outcome: Outcome
|
||||
) -> dict[str, Any]:
|
||||
base = {"role": role, "kind": kind.value, "attempt": attempt}
|
||||
if isinstance(outcome, Submitted):
|
||||
return {
|
||||
**base,
|
||||
"type": "submitted",
|
||||
"content": outcome.content,
|
||||
"attachments": dict(outcome.attachments),
|
||||
}
|
||||
if isinstance(outcome, NeedsInput):
|
||||
return {
|
||||
**base,
|
||||
"type": "needs_input",
|
||||
"questions": list(outcome.questions),
|
||||
"permission": outcome.permission,
|
||||
}
|
||||
if isinstance(outcome, BudgetExhausted):
|
||||
return {**base, "type": "budget_exhausted", "summary": outcome.summary}
|
||||
return {**base, "type": "stopped", "summary": _describe_unfinished(outcome)}
|
||||
|
||||
|
||||
def _describe_unfinished(outcome: Outcome) -> str:
|
||||
if isinstance(outcome, BudgetExhausted):
|
||||
return f"I paused: {outcome.summary} How should I proceed?"
|
||||
if isinstance(outcome, Stopped):
|
||||
return outcome.message
|
||||
if isinstance(outcome, NeedsInput):
|
||||
return " ".join(outcome.questions)
|
||||
return f"Ended with an unexpected outcome: {type(outcome).__name__}."
|
||||
|
||||
|
||||
def _automatic_reply(
|
||||
prompt: Mapping[str, Any], scripted_answers: Sequence[Sequence[str]] = ()
|
||||
) -> dict[str, Any]:
|
||||
"""The reply in a pre-approved Execution: approve, deny cloud reads, and answer
|
||||
questions from scripted (pattern, answer) pairs where one matches."""
|
||||
if "approval" in prompt:
|
||||
return {
|
||||
"approval": {"phase": prompt["approval"]["phase"], "decision": "approve"}
|
||||
}
|
||||
if "permission" in prompt:
|
||||
return {"permission": {prompt["permission"]["tier"]: False}}
|
||||
lines = []
|
||||
unanswered = False
|
||||
for item in prompt.get("questions", []):
|
||||
answer = next(
|
||||
(
|
||||
scripted
|
||||
for pattern, scripted in scripted_answers
|
||||
if re.search(pattern, item["question"], re.IGNORECASE)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if answer is None:
|
||||
unanswered = True
|
||||
else:
|
||||
lines.append(f"Q: {item['question']}\nA: {answer}")
|
||||
if unanswered or not lines:
|
||||
lines.append(NO_USER_REPLY)
|
||||
return {"reply": "\n\n".join(lines)}
|
||||
|
||||
|
||||
def _reply_text(reply: object) -> str:
|
||||
if isinstance(reply, str):
|
||||
return reply
|
||||
if isinstance(reply, Mapping) and isinstance(reply.get("reply"), str):
|
||||
return reply["reply"]
|
||||
return str(reply)
|
||||
|
||||
|
||||
def _render_prompt(prompt: Mapping[str, Any]) -> str:
|
||||
if "approval" in prompt:
|
||||
approval = prompt["approval"]
|
||||
return f"Please approve {approval['phase']}: {approval['summary']}"
|
||||
if "permission" in prompt:
|
||||
return f"May I use {prompt['permission']['tier']} tools?"
|
||||
return "\n".join(item["question"] for item in prompt.get("questions", []))
|
||||
|
||||
|
||||
def _write_outputs(
|
||||
out_dir: Path, deliverables: Mapping[str, Mapping[str, Any]]
|
||||
) -> None:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
guide = deliverables.get(DeliverableKind.SOLUTION_GUIDE.value)
|
||||
if guide:
|
||||
(out_dir / GUIDE_FILENAME).write_text(guide["content"], encoding="utf-8")
|
||||
for kind, deliverable in deliverables.items():
|
||||
path = out_dir / "deliverables" / f"{kind}.md"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(deliverable["content"], encoding="utf-8")
|
||||
for name, content in deliverable.get("attachments", {}).items():
|
||||
attachment = out_dir / name
|
||||
attachment.parent.mkdir(parents=True, exist_ok=True)
|
||||
attachment.write_text(content, encoding="utf-8")
|
||||
@@ -1,151 +0,0 @@
|
||||
"""Phase 3: Solution Guide Packaging Node."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
from app.states.state import GCPArchitectureState
|
||||
from app.skills.loader import SkillLoader
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def package_node(state: GCPArchitectureState, skill_loader: SkillLoader) -> Dict[str, Any]:
|
||||
"""Processes Phase 3 Packaging: assemble final solution-architecture-guide.md document."""
|
||||
logger.info("Executing package_node")
|
||||
skill_prompt = skill_loader.format_skills_for_prompt("package")
|
||||
|
||||
src_doc = state.get("source_discovery_doc", "")
|
||||
src_mmd = state.get("source_mermaid_diagram", "")
|
||||
req_doc = state.get("requirements_doc", "")
|
||||
arch_doc = state.get("architecture_doc", "")
|
||||
mmd_doc = state.get("mermaid_diagram", "")
|
||||
tf_doc = state.get("terraform_code", "")
|
||||
val_doc = state.get("validation_results", "")
|
||||
|
||||
solution_guide = f"""# Google Cloud solution architecture: Event-Driven Regional Workload
|
||||
|
||||
## 1. Executive summary and workload overview
|
||||
This document serves as the comprehensive reference architecture guide for migrating an event-driven application from a legacy pre-existing environment to a highly available, serverless Google Cloud architecture.
|
||||
|
||||
## 2. Requirements and current state
|
||||
|
||||
### 2.1. Functional requirements
|
||||
See [`docs/requirements.md`](docs/requirements.md). Requirements include Functional requirements, Non-functional requirements, constraints, assumptions, and open questions.
|
||||
- Accept authenticated HTTPS requests from external clients.
|
||||
- Execute stateless application logic behind a versioned service endpoint.
|
||||
- Asynchronously publish domain events to Pub/Sub.
|
||||
- Retain raw payload records in Cloud Storage for audit and replay.
|
||||
|
||||
### 2.2. Non-functional requirements
|
||||
- **Security**: HTTPS TLS 1.3 encryption in transit, managed encryption at rest, least-privilege IAM service accounts.
|
||||
- **Reliability**: 99.9% monthly endpoint availability target, regional high availability, Pub/Sub dead-letter topics.
|
||||
- **Cost**: Serverless pay-per-use scaling from zero instances to reduce idle compute expense.
|
||||
- **Operations**: Centralized logging via Cloud Logging and metrics via Cloud Monitoring.
|
||||
- **Performance**: Sub-500ms p95 latency for ingestion acknowledgements under peak load.
|
||||
- **Sustainability**: Efficient resource utilization via auto-scaling serverless runtimes.
|
||||
|
||||
### 2.3. Current state (As-Is Architecture)
|
||||
{src_doc.strip()}
|
||||
|
||||
```mermaid
|
||||
{(src_mmd or "flowchart TD\n Client --> LegacyApp").strip()}
|
||||
```
|
||||
|
||||
### 2.4. Dependencies
|
||||
- **Internal dependencies**: Service identity bindings and event consumer subscribers.
|
||||
- **External dependencies**: Client HTTP submitters and OCI container image registry.
|
||||
|
||||
## 3. Technical decomposition of the workload
|
||||
- **Ingress & Compute Layer**: Cloud Run service processing stateless HTTP webhook calls.
|
||||
- **Messaging & Decoupling Layer**: Pub/Sub topic buffering domain event messages.
|
||||
- **Storage & Audit Layer**: Cloud Storage buckets for raw payload audit log retention and Firestore for structured document state.
|
||||
|
||||
## 4. Proposed solution architecture
|
||||
|
||||
### 4.1. Google Cloud products and features mapping (Selected products)
|
||||
| Component | Recommended Google Cloud product/feature | Justification and citations | Alternatives considered | Pros and cons of alternatives |
|
||||
| :--- | :--- | :--- | :--- | :--- |
|
||||
| **Compute** | **Google Cloud Run** | Fully managed serverless execution with auto-scaling to zero ([Cloud Run Docs](https://cloud.google.com/run/docs/overview)) | GKE / Compute Engine MIGs | **Pros**: Granular cluster control <br> **Cons**: Higher operational overhead & idle costs |
|
||||
| **Messaging** | **Google Cloud Pub/Sub** | Asynchronous regional event bus with at-least-once delivery ([Pub/Sub Docs](https://cloud.google.com/pubsub/docs/overview)) | Cloud Tasks / Kafka | **Pros**: Advanced queuing controls <br> **Cons**: Complex cluster management |
|
||||
| **Storage** | **Google Cloud Storage & Firestore** | Durable object retention with lifecycle rules & NoSQL document database | Cloud SQL | **Pros**: Relational ACID support <br> **Cons**: Less flexible scaling for unstructured event logs |
|
||||
|
||||
### 4.2. Architecture diagram (Mermaid)
|
||||
```mermaid
|
||||
{mmd_doc.strip()}
|
||||
```
|
||||
|
||||
### 4.3. Architecture description
|
||||
- **Data flow**: Clients send HTTPS requests to Cloud Run -> Cloud Run writes payload to Cloud Storage & publishes event to Pub/Sub -> Subscriber worker consumes event.
|
||||
- **Tasks/control flow**: Client request -> Token validation -> Pub/Sub acknowledgement -> Async worker trigger.
|
||||
|
||||
## 5. Design and configuration recommendations
|
||||
|
||||
### 5.1. Security, privacy, and compliance
|
||||
- **Access control**: Least-privilege IAM service accounts bound to publisher roles.
|
||||
- **Data protection**: Managed encryption at rest for Pub/Sub and Storage.
|
||||
- **Network Security**: Serverless VPC Access connector for isolated network egress.
|
||||
|
||||
### 5.2. Reliability
|
||||
- **Redundant deployment**: Regional Cloud Run service and Pub/Sub multi-zone replication.
|
||||
- **Backup and DR**: Cross-region bucket replication and dead-letter retry topic.
|
||||
|
||||
### 5.3. Operational excellence
|
||||
- **Monitoring and logging**: Integrated Cloud Logging and Cloud Monitoring alerts.
|
||||
- **Infrastructure as Code (IaC)**: Version-controlled Terraform HCL blueprints.
|
||||
|
||||
### 5.4. Cost optimization
|
||||
- **Sizing and scaling**: Automatic scale-to-zero compute instances.
|
||||
|
||||
### 5.5. Performance efficiency
|
||||
- **Caching and CDN**: Edge CDN caching for static endpoints.
|
||||
|
||||
### 5.6. Sustainability
|
||||
- Serverless compute adoption minimizing idle carbon footprint.
|
||||
|
||||
## 6. Deployment guidance
|
||||
|
||||
### 6.1. Deployment prerequisites
|
||||
- Enable required Google Cloud APIs (`run.googleapis.com`, `pubsub.googleapis.com`, `storage.googleapis.com`).
|
||||
- Install Terraform >= 1.5.0 and Google Cloud SDK (`gcloud`).
|
||||
|
||||
### 6.2. Step-by-step deployment instructions (Terraform)
|
||||
```hcl
|
||||
{tf_doc.strip()}
|
||||
```
|
||||
|
||||
Apply blueprint instructions:
|
||||
```bash
|
||||
terraform -chdir=terraform init
|
||||
terraform -chdir=terraform plan -var='project_id=YOUR_PROJECT_ID' -var='container_image=IMAGE_URI'
|
||||
terraform -chdir=terraform apply
|
||||
```
|
||||
|
||||
## 7. Validation plan (Validation results)
|
||||
{val_doc.strip()}
|
||||
|
||||
### Verification Checklist
|
||||
- Step 4 guide persistence: non-empty solution-architecture-guide.md.
|
||||
- Step 5 template/workflow conformance: verified requirements, architecture, Terraform, diagram.
|
||||
- Step 6 & 7 publication & remote verification: complete.
|
||||
|
||||
## 8. References
|
||||
- [Google Cloud Architecture Framework](https://cloud.google.com/architecture/framework)
|
||||
- [Cloud Run Overview](https://cloud.google.com/run/docs/overview/what-is-cloud-run)
|
||||
- [Cloud Pub/Sub Overview](https://cloud.google.com/pubsub/docs/overview)
|
||||
"""
|
||||
|
||||
active_skills = state.get("active_skills", [])
|
||||
if "packaging_guide" not in active_skills:
|
||||
active_skills.append("packaging_guide")
|
||||
|
||||
status_summary = {
|
||||
"phases_completed": ["discover", "design", "validate", "package"],
|
||||
"validation_passed": state.get("validation_passed", True),
|
||||
"total_active_skills": len(active_skills),
|
||||
}
|
||||
|
||||
return {
|
||||
"solution_guide": solution_guide,
|
||||
"current_phase": "package",
|
||||
"active_skills": active_skills,
|
||||
"status_summary": status_summary,
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
"""Phase 0a: Existing Source Environment Discovery Node."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict
|
||||
from app.states.state import GCPArchitectureState
|
||||
from app.skills.loader import SkillLoader
|
||||
from app.tools.gcp_scanner import GCPEnvironmentScanner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def source_discover_node(state: GCPArchitectureState, skill_loader: SkillLoader) -> Dict[str, Any]:
|
||||
"""Processes Phase 0a Source Environment Discovery: documents existing as-is architecture before target migration."""
|
||||
logger.info("Executing source_discover_node")
|
||||
skill_prompt = skill_loader.format_skills_for_prompt("source_discover")
|
||||
|
||||
# Extract target GCP project ID from request if specified
|
||||
request_summary = state.get("workflow_request", "")
|
||||
project_id = state.get("project_id")
|
||||
if not project_id and request_summary:
|
||||
match = re.search(r"project\s+([a-z0-9-]+)", request_summary, re.IGNORECASE)
|
||||
if match:
|
||||
project_id = match.group(1)
|
||||
|
||||
# Run live GCP environment scanner
|
||||
scanner = GCPEnvironmentScanner(project_id=project_id)
|
||||
scan_result = scanner.scan_environment()
|
||||
report = scanner.format_scan_report(scan_result)
|
||||
|
||||
source_discovery_doc = report["doc"]
|
||||
source_mermaid_diagram = report["mermaid"]
|
||||
|
||||
active_skills = state.get("active_skills", [])
|
||||
if "source_discovery" not in active_skills:
|
||||
active_skills.append("source_discovery")
|
||||
|
||||
return {
|
||||
"source_discovery_doc": source_discovery_doc,
|
||||
"source_mermaid_diagram": source_mermaid_diagram,
|
||||
"current_phase": "source_discover",
|
||||
"active_skills": active_skills,
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
"""Phase 2: Pre-deployment Validation Node."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
from app.states.state import GCPArchitectureState
|
||||
from app.skills.loader import SkillLoader
|
||||
from app.tools.validation_tools import (
|
||||
validate_mermaid_diagram,
|
||||
validate_terraform_syntax,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def validate_node(state: GCPArchitectureState, skill_loader: SkillLoader) -> Dict[str, Any]:
|
||||
"""Processes Phase 2 Validation: verify diagram, Terraform IaC, and artifact integrity."""
|
||||
logger.info("Executing validate_node")
|
||||
skill_prompt = skill_loader.format_skills_for_prompt("validate")
|
||||
|
||||
mermaid_content = state.get("mermaid_diagram", "")
|
||||
terraform_content = state.get("terraform_code", "")
|
||||
|
||||
mermaid_check = validate_mermaid_diagram.invoke({"diagram_content": mermaid_content})
|
||||
terraform_check = validate_terraform_syntax.invoke({"terraform_content": terraform_content})
|
||||
|
||||
errors = []
|
||||
if not mermaid_check.get("valid"):
|
||||
errors.append(f"Mermaid Check Failed: {mermaid_check.get('error')}")
|
||||
if not terraform_check.get("valid"):
|
||||
errors.append(f"Terraform Check Failed: {terraform_check.get('error')}")
|
||||
|
||||
validation_passed = len(errors) == 0
|
||||
|
||||
validation_results = f"""# Validation Results
|
||||
|
||||
## Summary
|
||||
- **Overall Validation Status**: {"PASS" if validation_passed else "FAIL"}
|
||||
- **Mermaid Diagram Syntax**: {"PASS" if mermaid_check.get("valid") else "FAIL"}
|
||||
- **Terraform Structural Check**: {"PASS" if terraform_check.get("valid") else "FAIL"}
|
||||
- **Resource Provisioning Triggered**: False (Static non-deployment check enforced)
|
||||
|
||||
## Verification Rules Checklist
|
||||
- [x] Functional & Non-functional requirements specified
|
||||
- [x] Product selection deferred during discovery and resolved in design phase
|
||||
- [x] Regional High Availability and Security IAM boundaries configured
|
||||
- [x] Mermaid diagram follows valid graph syntax
|
||||
- [x] Terraform HCL declares provider, resources, and least-privilege IAM bindings
|
||||
"""
|
||||
|
||||
active_skills = state.get("active_skills", [])
|
||||
if "validation_rules" not in active_skills:
|
||||
active_skills.append("validation_rules")
|
||||
|
||||
return {
|
||||
"validation_results": validation_results,
|
||||
"validation_passed": validation_passed,
|
||||
"errors": errors,
|
||||
"current_phase": "validate",
|
||||
"active_skills": active_skills,
|
||||
}
|
||||
Reference in New Issue
Block a user