Files
orchestration_engine/execution_engine/providers.py

290 lines
11 KiB
Python

"""Cloud provider bundles.
A Cloud provider is configuration, not code (ADR-0002): providers/<name>.yaml lists
the Skills, local tools, remote MCP servers and default models that one Cloud
provider brings. Nothing outside this module names a provider.
"""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import yaml
from execution_engine.scope import Catalogue, load_catalogue
from execution_engine.skills.agent_skills import (
SkillFormatError,
SkillMeta,
discover_skills,
)
TIERS = ("general", "cloud_read")
_REPO_ROOT = Path(__file__).resolve().parent.parent
class ProviderConfigError(ValueError):
"""A provider bundle is missing or invalid."""
@dataclass(frozen=True)
class McpServerSpec:
"""A remote MCP server hosted by the Cloud provider.
Only tools named in `allow` are ever exposed; an empty `allow` exposes nothing.
"""
id: str
url: str
tier: str
allow: tuple[str, ...]
preview: bool = False
@dataclass(frozen=True)
class ModelConfig:
default: str
roles: Mapping[str, str] = field(default_factory=dict)
# Seconds one model request may take, for the roles whose replies are long. A
# role not named here gets the runtime's default.
timeouts: Mapping[str, float] = field(default_factory=dict)
# A lighter model for a role's first attempt at a Deliverable. Only for a role
# whose work something else checks; an attempt after feedback takes its own.
first_attempt: Mapping[str, str] = field(default_factory=dict)
def for_role(self, role: str) -> str:
return self.roles.get(role, self.default)
def timeout_for(self, role: str) -> float | None:
return self.timeouts.get(role)
@dataclass(frozen=True)
class ProviderBundle:
name: str
skills_dirs: tuple[Path, ...]
skills: Mapping[str, SkillMeta]
# The template a Workflow's final Deliverable follows, when it has one.
output_template: Path | None
models: ModelConfig
tools: Mapping[str, tuple[str, ...]]
mcp_auth: Mapping[str, Any]
mcp_servers: Mapping[str, McpServerSpec]
role_mcp_servers: Mapping[str, tuple[str, ...]]
role_skills: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
pillar_skills: Mapping[str, str] = field(default_factory=dict)
# The Terraform providers generated code is written for, pinned so the schema
# lookup, the patterns and validation agree: {name: {source, version}}.
terraform_providers: Mapping[str, Mapping[str, str]] = field(default_factory=dict)
# Registry namespaces generated code may take a Terraform module from. Empty
# means no module is accepted.
terraform_module_namespaces: tuple[str, ...] = ()
# The Skill that says which modules to use and when. It reaches a Specialist
# only in an Execution that opted into modules.
terraform_modules_skill: str | None = None
# Modules that could not be used with the pinned providers when last looked
# at, as <namespace>/<name>/<provider>. scripts/check-terraform-modules.py
# says when one can.
terraform_modules_watch: tuple[str, ...] = ()
# The Skill approved lessons are published in (ADR-0004); None when the provider
# has none. A Specialist is given it once it holds lessons for its Deliverable.
lessons_skill: str | None = None
# The Skill that holds the scope spec's catalogue (ADR-0008), and the values
# this provider allows for the fields the catalogue leaves to a provider.
scope_skill: str | None = None
scope_values: Mapping[str, tuple[str, ...]] = field(default_factory=dict)
def scope_catalogue(self) -> Catalogue:
if not self.scope_skill:
raise ProviderConfigError(f"The {self.name} bundle names no scope Skill.")
return load_catalogue(
self.skills[self.scope_skill].path / "assets" / "catalogue.yaml"
)
def mcp_servers_for_role(
self, role: str, include_preview: bool = True
) -> tuple[McpServerSpec, ...]:
servers = (
self.mcp_servers[server_id]
for server_id in self.role_mcp_servers.get(role, ())
)
return tuple(
server for server in servers if include_preview or not server.preview
)
def load_provider(name: str, root: Path | None = None) -> ProviderBundle:
"""Load and validate providers/<name>.yaml; relative paths resolve against `root`."""
root = root or _REPO_ROOT
config_path = root / "providers" / f"{name}.yaml"
if not config_path.is_file():
raise ProviderConfigError(f"No provider bundle for {name!r} at {config_path}")
config = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
if not isinstance(config, dict):
raise ProviderConfigError(f"{config_path}: must be a mapping")
def fail(message: str) -> ProviderConfigError:
return ProviderConfigError(f"{config_path}: {message}")
if config.get("name") != name:
raise fail(f"'name' must be {name!r}")
skills_dirs = tuple(root / entry for entry in config.get("skills_dirs") or ())
if not skills_dirs:
raise fail("'skills_dirs' must list at least one directory")
for directory in skills_dirs:
if not directory.is_dir():
raise fail(f"'skills_dirs' entry {directory} is not a directory")
try:
skills = discover_skills(skills_dirs)
except SkillFormatError as exc:
raise fail(str(exc)) from exc
output_template: Path | None = None
if config.get("output_template"):
output_template = root / str(config["output_template"])
if not output_template.is_file():
raise fail(f"'output_template' {output_template} does not exist")
models = config.get("models") or {}
if not isinstance(models.get("default"), str) or not models["default"]:
raise fail("'models.default' is required")
timeouts = {}
for role, seconds in (models.get("timeout_seconds") or {}).items():
if isinstance(seconds, bool) or not isinstance(seconds, (int, float)):
raise fail(f"'models.timeout_seconds' for {role!r} must be a number")
if seconds <= 0:
raise fail(f"'models.timeout_seconds' for {role!r} must be positive")
timeouts[str(role)] = float(seconds)
model_config = ModelConfig(
default=models["default"],
roles=dict(models.get("roles") or {}),
timeouts=timeouts,
first_attempt={
str(role): str(model)
for role, model in (models.get("first_attempt") or {}).items()
},
)
terraform_providers = {}
pins = (config.get("terraform") or {}).get("required_providers") or {}
for provider, pin in pins.items():
if not isinstance(pin, dict) or not all(
isinstance(pin.get(key), str) and pin[key] for key in ("source", "version")
):
raise fail(
f"terraform provider {provider!r} needs a 'source' and an exact "
"'version', both strings"
)
terraform_providers[str(provider)] = {
"source": pin["source"],
"version": pin["version"],
}
module_namespaces = ((config.get("terraform") or {}).get("modules") or {}).get(
"namespaces"
) or []
if not isinstance(module_namespaces, list) or not all(
isinstance(item, str) and item for item in module_namespaces
):
raise fail("'terraform.modules.namespaces' must be a list of names")
modules_skill = ((config.get("terraform") or {}).get("modules") or {}).get(
"skill"
) or None
if modules_skill is not None and modules_skill not in skills:
raise fail(
f"'terraform.modules.skill' references unknown skill {modules_skill!r}"
)
tools = {}
for tier, tool_names in (config.get("tools") or {}).items():
if tier not in TIERS:
raise fail(f"'tools' tier {tier!r} must be one of {TIERS}")
tools[tier] = tuple(tool_names or ())
mcp_servers = {}
for server_id, spec in (config.get("mcp_servers") or {}).items():
url = spec.get("url", "")
if not url.startswith("https://"):
raise fail(f"MCP server {server_id!r} url must use https")
if spec.get("tier") not in TIERS:
raise fail(f"MCP server {server_id!r} tier must be one of {TIERS}")
if not isinstance(spec.get("allow"), list):
raise fail(
f"MCP server {server_id!r} needs an 'allow' list (empty exposes no tools)"
)
mcp_servers[server_id] = McpServerSpec(
id=server_id,
url=url,
tier=spec["tier"],
allow=tuple(spec["allow"]),
preview=bool(spec.get("preview", False)),
)
role_mcp_servers = {}
for role, server_ids in (config.get("role_mcp_servers") or {}).items():
unknown = [
server_id for server_id in server_ids or () if server_id not in mcp_servers
]
if unknown:
raise fail(
f"role {role!r} references unknown MCP servers: {', '.join(unknown)}"
)
role_mcp_servers[role] = tuple(server_ids or ())
role_skills = {}
for role, skill_names in (config.get("role_skills") or {}).items():
unknown = [skill for skill in skill_names or () if skill not in skills]
if unknown:
raise fail(f"role {role!r} references unknown skills: {', '.join(unknown)}")
role_skills[role] = tuple(skill_names or ())
pillar_skills = {}
for pillar, skill_name in (config.get("pillar_skills") or {}).items():
if skill_name not in skills:
raise fail(f"pillar {pillar!r} references unknown skill {skill_name!r}")
pillar_skills[str(pillar)] = skill_name
lessons_skill = config.get("lessons_skill") or None
if lessons_skill is not None and lessons_skill not in skills:
raise fail(f"'lessons_skill' references unknown skill {lessons_skill!r}")
scope = config.get("scope") or {}
scope_skill = scope.get("skill") or None
if scope_skill is not None and scope_skill not in skills:
raise fail(f"'scope.skill' references unknown skill {scope_skill!r}")
scope_values = {
str(path): tuple(str(value) for value in values or ())
for path, values in (scope.get("values") or {}).items()
}
return ProviderBundle(
name=name,
skills_dirs=skills_dirs,
skills=skills,
output_template=output_template,
models=model_config,
tools=tools,
mcp_auth=dict(config.get("mcp_auth") or {}),
mcp_servers=mcp_servers,
role_mcp_servers=role_mcp_servers,
role_skills=role_skills,
pillar_skills=pillar_skills,
terraform_providers=terraform_providers,
terraform_module_namespaces=tuple(module_namespaces),
terraform_modules_skill=modules_skill,
terraform_modules_watch=tuple(
str(item)
for item in ((config.get("terraform") or {}).get("modules") or {}).get(
"watch"
)
or ()
),
lessons_skill=lessons_skill,
scope_skill=scope_skill,
scope_values=scope_values,
)