feat: working orhcestration engine
This commit is contained in:
289
execution_engine/providers.py
Normal file
289
execution_engine/providers.py
Normal file
@@ -0,0 +1,289 @@
|
||||
"""Cloud provider bundles.
|
||||
|
||||
A Cloud provider is configuration, not code (ADR-0002): providers/<name>.yaml lists
|
||||
the Skills, local tools, remote MCP servers and default models that one Cloud
|
||||
provider brings. Nothing outside this module names a provider.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from execution_engine.scope import Catalogue, load_catalogue
|
||||
from execution_engine.skills.agent_skills import (
|
||||
SkillFormatError,
|
||||
SkillMeta,
|
||||
discover_skills,
|
||||
)
|
||||
|
||||
TIERS = ("general", "cloud_read")
|
||||
_REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
class ProviderConfigError(ValueError):
|
||||
"""A provider bundle is missing or invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class McpServerSpec:
|
||||
"""A remote MCP server hosted by the Cloud provider.
|
||||
|
||||
Only tools named in `allow` are ever exposed; an empty `allow` exposes nothing.
|
||||
"""
|
||||
|
||||
id: str
|
||||
url: str
|
||||
tier: str
|
||||
allow: tuple[str, ...]
|
||||
preview: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModelConfig:
|
||||
default: str
|
||||
roles: Mapping[str, str] = field(default_factory=dict)
|
||||
# Seconds one model request may take, for the roles whose replies are long. A
|
||||
# role not named here gets the runtime's default.
|
||||
timeouts: Mapping[str, float] = field(default_factory=dict)
|
||||
# A lighter model for a role's first attempt at a Deliverable. Only for a role
|
||||
# whose work something else checks; an attempt after feedback takes its own.
|
||||
first_attempt: Mapping[str, str] = field(default_factory=dict)
|
||||
|
||||
def for_role(self, role: str) -> str:
|
||||
return self.roles.get(role, self.default)
|
||||
|
||||
def timeout_for(self, role: str) -> float | None:
|
||||
return self.timeouts.get(role)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderBundle:
|
||||
name: str
|
||||
skills_dirs: tuple[Path, ...]
|
||||
skills: Mapping[str, SkillMeta]
|
||||
# The template a Workflow's final Deliverable follows, when it has one.
|
||||
output_template: Path | None
|
||||
models: ModelConfig
|
||||
tools: Mapping[str, tuple[str, ...]]
|
||||
mcp_auth: Mapping[str, Any]
|
||||
mcp_servers: Mapping[str, McpServerSpec]
|
||||
role_mcp_servers: Mapping[str, tuple[str, ...]]
|
||||
role_skills: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
pillar_skills: Mapping[str, str] = field(default_factory=dict)
|
||||
# The Terraform providers generated code is written for, pinned so the schema
|
||||
# lookup, the patterns and validation agree: {name: {source, version}}.
|
||||
terraform_providers: Mapping[str, Mapping[str, str]] = field(default_factory=dict)
|
||||
# Registry namespaces generated code may take a Terraform module from. Empty
|
||||
# means no module is accepted.
|
||||
terraform_module_namespaces: tuple[str, ...] = ()
|
||||
# The Skill that says which modules to use and when. It reaches a Specialist
|
||||
# only in an Execution that opted into modules.
|
||||
terraform_modules_skill: str | None = None
|
||||
# Modules that could not be used with the pinned providers when last looked
|
||||
# at, as <namespace>/<name>/<provider>. scripts/check-terraform-modules.py
|
||||
# says when one can.
|
||||
terraform_modules_watch: tuple[str, ...] = ()
|
||||
# The Skill approved lessons are published in (ADR-0004); None when the provider
|
||||
# has none. A Specialist is given it once it holds lessons for its Deliverable.
|
||||
lessons_skill: str | None = None
|
||||
# The Skill that holds the scope spec's catalogue (ADR-0008), and the values
|
||||
# this provider allows for the fields the catalogue leaves to a provider.
|
||||
scope_skill: str | None = None
|
||||
scope_values: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
|
||||
|
||||
def scope_catalogue(self) -> Catalogue:
|
||||
if not self.scope_skill:
|
||||
raise ProviderConfigError(f"The {self.name} bundle names no scope Skill.")
|
||||
return load_catalogue(
|
||||
self.skills[self.scope_skill].path / "assets" / "catalogue.yaml"
|
||||
)
|
||||
|
||||
def mcp_servers_for_role(
|
||||
self, role: str, include_preview: bool = True
|
||||
) -> tuple[McpServerSpec, ...]:
|
||||
servers = (
|
||||
self.mcp_servers[server_id]
|
||||
for server_id in self.role_mcp_servers.get(role, ())
|
||||
)
|
||||
return tuple(
|
||||
server for server in servers if include_preview or not server.preview
|
||||
)
|
||||
|
||||
|
||||
def load_provider(name: str, root: Path | None = None) -> ProviderBundle:
|
||||
"""Load and validate providers/<name>.yaml; relative paths resolve against `root`."""
|
||||
root = root or _REPO_ROOT
|
||||
config_path = root / "providers" / f"{name}.yaml"
|
||||
if not config_path.is_file():
|
||||
raise ProviderConfigError(f"No provider bundle for {name!r} at {config_path}")
|
||||
|
||||
config = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
|
||||
if not isinstance(config, dict):
|
||||
raise ProviderConfigError(f"{config_path}: must be a mapping")
|
||||
|
||||
def fail(message: str) -> ProviderConfigError:
|
||||
return ProviderConfigError(f"{config_path}: {message}")
|
||||
|
||||
if config.get("name") != name:
|
||||
raise fail(f"'name' must be {name!r}")
|
||||
|
||||
skills_dirs = tuple(root / entry for entry in config.get("skills_dirs") or ())
|
||||
if not skills_dirs:
|
||||
raise fail("'skills_dirs' must list at least one directory")
|
||||
for directory in skills_dirs:
|
||||
if not directory.is_dir():
|
||||
raise fail(f"'skills_dirs' entry {directory} is not a directory")
|
||||
try:
|
||||
skills = discover_skills(skills_dirs)
|
||||
except SkillFormatError as exc:
|
||||
raise fail(str(exc)) from exc
|
||||
|
||||
output_template: Path | None = None
|
||||
if config.get("output_template"):
|
||||
output_template = root / str(config["output_template"])
|
||||
if not output_template.is_file():
|
||||
raise fail(f"'output_template' {output_template} does not exist")
|
||||
|
||||
models = config.get("models") or {}
|
||||
if not isinstance(models.get("default"), str) or not models["default"]:
|
||||
raise fail("'models.default' is required")
|
||||
timeouts = {}
|
||||
for role, seconds in (models.get("timeout_seconds") or {}).items():
|
||||
if isinstance(seconds, bool) or not isinstance(seconds, (int, float)):
|
||||
raise fail(f"'models.timeout_seconds' for {role!r} must be a number")
|
||||
if seconds <= 0:
|
||||
raise fail(f"'models.timeout_seconds' for {role!r} must be positive")
|
||||
timeouts[str(role)] = float(seconds)
|
||||
model_config = ModelConfig(
|
||||
default=models["default"],
|
||||
roles=dict(models.get("roles") or {}),
|
||||
timeouts=timeouts,
|
||||
first_attempt={
|
||||
str(role): str(model)
|
||||
for role, model in (models.get("first_attempt") or {}).items()
|
||||
},
|
||||
)
|
||||
|
||||
terraform_providers = {}
|
||||
pins = (config.get("terraform") or {}).get("required_providers") or {}
|
||||
for provider, pin in pins.items():
|
||||
if not isinstance(pin, dict) or not all(
|
||||
isinstance(pin.get(key), str) and pin[key] for key in ("source", "version")
|
||||
):
|
||||
raise fail(
|
||||
f"terraform provider {provider!r} needs a 'source' and an exact "
|
||||
"'version', both strings"
|
||||
)
|
||||
terraform_providers[str(provider)] = {
|
||||
"source": pin["source"],
|
||||
"version": pin["version"],
|
||||
}
|
||||
|
||||
module_namespaces = ((config.get("terraform") or {}).get("modules") or {}).get(
|
||||
"namespaces"
|
||||
) or []
|
||||
if not isinstance(module_namespaces, list) or not all(
|
||||
isinstance(item, str) and item for item in module_namespaces
|
||||
):
|
||||
raise fail("'terraform.modules.namespaces' must be a list of names")
|
||||
modules_skill = ((config.get("terraform") or {}).get("modules") or {}).get(
|
||||
"skill"
|
||||
) or None
|
||||
if modules_skill is not None and modules_skill not in skills:
|
||||
raise fail(
|
||||
f"'terraform.modules.skill' references unknown skill {modules_skill!r}"
|
||||
)
|
||||
|
||||
tools = {}
|
||||
for tier, tool_names in (config.get("tools") or {}).items():
|
||||
if tier not in TIERS:
|
||||
raise fail(f"'tools' tier {tier!r} must be one of {TIERS}")
|
||||
tools[tier] = tuple(tool_names or ())
|
||||
|
||||
mcp_servers = {}
|
||||
for server_id, spec in (config.get("mcp_servers") or {}).items():
|
||||
url = spec.get("url", "")
|
||||
if not url.startswith("https://"):
|
||||
raise fail(f"MCP server {server_id!r} url must use https")
|
||||
if spec.get("tier") not in TIERS:
|
||||
raise fail(f"MCP server {server_id!r} tier must be one of {TIERS}")
|
||||
if not isinstance(spec.get("allow"), list):
|
||||
raise fail(
|
||||
f"MCP server {server_id!r} needs an 'allow' list (empty exposes no tools)"
|
||||
)
|
||||
mcp_servers[server_id] = McpServerSpec(
|
||||
id=server_id,
|
||||
url=url,
|
||||
tier=spec["tier"],
|
||||
allow=tuple(spec["allow"]),
|
||||
preview=bool(spec.get("preview", False)),
|
||||
)
|
||||
|
||||
role_mcp_servers = {}
|
||||
for role, server_ids in (config.get("role_mcp_servers") or {}).items():
|
||||
unknown = [
|
||||
server_id for server_id in server_ids or () if server_id not in mcp_servers
|
||||
]
|
||||
if unknown:
|
||||
raise fail(
|
||||
f"role {role!r} references unknown MCP servers: {', '.join(unknown)}"
|
||||
)
|
||||
role_mcp_servers[role] = tuple(server_ids or ())
|
||||
|
||||
role_skills = {}
|
||||
for role, skill_names in (config.get("role_skills") or {}).items():
|
||||
unknown = [skill for skill in skill_names or () if skill not in skills]
|
||||
if unknown:
|
||||
raise fail(f"role {role!r} references unknown skills: {', '.join(unknown)}")
|
||||
role_skills[role] = tuple(skill_names or ())
|
||||
|
||||
pillar_skills = {}
|
||||
for pillar, skill_name in (config.get("pillar_skills") or {}).items():
|
||||
if skill_name not in skills:
|
||||
raise fail(f"pillar {pillar!r} references unknown skill {skill_name!r}")
|
||||
pillar_skills[str(pillar)] = skill_name
|
||||
|
||||
lessons_skill = config.get("lessons_skill") or None
|
||||
if lessons_skill is not None and lessons_skill not in skills:
|
||||
raise fail(f"'lessons_skill' references unknown skill {lessons_skill!r}")
|
||||
|
||||
scope = config.get("scope") or {}
|
||||
scope_skill = scope.get("skill") or None
|
||||
if scope_skill is not None and scope_skill not in skills:
|
||||
raise fail(f"'scope.skill' references unknown skill {scope_skill!r}")
|
||||
scope_values = {
|
||||
str(path): tuple(str(value) for value in values or ())
|
||||
for path, values in (scope.get("values") or {}).items()
|
||||
}
|
||||
|
||||
return ProviderBundle(
|
||||
name=name,
|
||||
skills_dirs=skills_dirs,
|
||||
skills=skills,
|
||||
output_template=output_template,
|
||||
models=model_config,
|
||||
tools=tools,
|
||||
mcp_auth=dict(config.get("mcp_auth") or {}),
|
||||
mcp_servers=mcp_servers,
|
||||
role_mcp_servers=role_mcp_servers,
|
||||
role_skills=role_skills,
|
||||
pillar_skills=pillar_skills,
|
||||
terraform_providers=terraform_providers,
|
||||
terraform_module_namespaces=tuple(module_namespaces),
|
||||
terraform_modules_skill=modules_skill,
|
||||
terraform_modules_watch=tuple(
|
||||
str(item)
|
||||
for item in ((config.get("terraform") or {}).get("modules") or {}).get(
|
||||
"watch"
|
||||
)
|
||||
or ()
|
||||
),
|
||||
lessons_skill=lessons_skill,
|
||||
scope_skill=scope_skill,
|
||||
scope_values=scope_values,
|
||||
)
|
||||
Reference in New Issue
Block a user