Runner: run a voyage remotely as the kab-automation identity
- shakedown_lib: runs F01 -> F02 -> F03 for one engagement on KAB with no personal token. Signs in as the kab-automation client (client credentials, provisioned by Stellwerk from crucible-platform-deploy) and renews the token itself. - Each stage checks its workflow id against the registered sub-agent name, builds inputs from the case and upstream stages, starts the run, follows it and judges it with cp1-agents evaluate_case.py. - Gates (plan 5b): in config/runner.yaml gate_projects only, continue on a critic pass with no failed step and a note saying the system decided. Never on a fail. Elsewhere, wait for a person and record who decided. gate_projects is empty for now. - Ledger per voyage run, with --resume; nothing secret is written. - Cymbal voyage ENG-CYMBAL-001; workflow registry; Dockerfile and an unapplied Job template; 15 tests against a fake KAB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
389
shakedown_lib/runner.py
Normal file
389
shakedown_lib/runner.py
Normal file
@@ -0,0 +1,389 @@
|
||||
"""Run a voyage remotely, stage by stage, with no person in the loop except where the rules keep one.
|
||||
|
||||
For each stage, in order:
|
||||
1. **Start**: check the workflow id still names the registered workflow; build the inputs from the context file and
|
||||
the voyage (upstream ids resolved); check required inputs and the engagement id; start the run.
|
||||
A stage with `use_execution` adopts an existing, finished run instead.
|
||||
2. **Follow**: poll the run. At a review point:
|
||||
- in a project listed in `config/runner.yaml` `gate_projects`, the gate rule (gate.py) decides: continue only on a
|
||||
critic `pass` with no failed step, with a note saying the system decided. Otherwise the pause is left for a
|
||||
person and the voyage stops for a fix;
|
||||
- in any other project, wait for a person to decide in KAB.
|
||||
3. **Finish**: pull the outputs, find the package and its artifact id, judge the run against its case
|
||||
(cp1-agents `evaluate_case.py`), and stop the voyage on a failed run, a missing package or a failed critical check.
|
||||
|
||||
The ledger is written on every change, so `--resume` picks a voyage up where it stopped (a started run is followed,
|
||||
never started twice).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
import yaml
|
||||
|
||||
from . import cp1_agents, gate
|
||||
from .kab import Kab, KabError
|
||||
from .ledger import Ledger, shown_inputs, utc_iso
|
||||
from .voyage import Stage, Voyage, VoyageError, Workflow, load_workflow, resolve_path
|
||||
|
||||
DONE_OK = {"completed", "succeeded", "success"}
|
||||
DONE_BAD = {"failed", "error", "cancelled", "canceled", "aborted", "stopped", "rejected", "timed_out", "timeout"}
|
||||
PLACEHOLDER = re.compile(r"^<[^>]+>$")
|
||||
|
||||
EXIT_OK, EXIT_CONFIG, EXIT_STOPPED, EXIT_PERSON_TIMEOUT, EXIT_TIMEOUT = 0, 2, 3, 4, 5
|
||||
|
||||
|
||||
class Stop(Exception):
|
||||
def __init__(self, code: int, reason: str) -> None:
|
||||
super().__init__(reason)
|
||||
self.code = code
|
||||
self.reason = reason
|
||||
|
||||
|
||||
@dataclass
|
||||
class Settings:
|
||||
gate_projects: set
|
||||
poll_seconds: int = 30
|
||||
stage_timeout_s: int = 6 * 3600
|
||||
person_wait_s: int = 24 * 3600
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict) -> "Settings":
|
||||
return cls(gate_projects=set(config.get("gate_projects") or set()),
|
||||
poll_seconds=int(config.get("poll_seconds", 30)),
|
||||
stage_timeout_s=int(config.get("stage_timeout_minutes", 360)) * 60,
|
||||
person_wait_s=int(config.get("person_wait_hours", 24)) * 3600)
|
||||
|
||||
|
||||
Evaluator = Callable[[str, dict, Any], tuple]
|
||||
|
||||
|
||||
def default_evaluator(feature: str, case: dict, steps: Any) -> tuple:
|
||||
return cp1_agents.evaluate_case().evaluate(feature, case, steps)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Runner:
|
||||
kab: Kab
|
||||
ledger: Ledger
|
||||
voyage: Voyage
|
||||
settings: Settings
|
||||
evaluator: Evaluator = default_evaluator
|
||||
clock: Callable[[], float] = time.time
|
||||
sleep: Callable[[float], None] = time.sleep
|
||||
log: Callable[[str], None] = print
|
||||
state: dict = field(default_factory=dict)
|
||||
|
||||
# ---- state -------------------------------------------------------------------------------------------------
|
||||
def _load(self) -> None:
|
||||
self.state = self.ledger.load_state() or {
|
||||
"voyage": self.voyage.name, "engagement_id": self.voyage.engagement_id, "project": self.voyage.project,
|
||||
"voyage_file": str(self.voyage.source) if self.voyage.source else None, "started_at": utc_iso(self.clock()),
|
||||
"cp1_agents_commit": cp1_agents.commit(), "decided_by": "kab-automation (Shakedown runner)",
|
||||
"stages": {}}
|
||||
|
||||
def _save(self) -> None:
|
||||
self.state["updated_at"] = utc_iso(self.clock())
|
||||
self.ledger.save_state(self.state)
|
||||
|
||||
def _stage_state(self, key: str) -> dict:
|
||||
return self.state["stages"].setdefault(key, {"state": "pending", "decisions": []})
|
||||
|
||||
# ---- the voyage --------------------------------------------------------------------------------------------
|
||||
def run(self, dry_run: bool = False, only: Optional[List[str]] = None) -> int:
|
||||
self._load()
|
||||
if self.state.get("project") != self.voyage.project:
|
||||
self.log(f"refused: the ledger is for project {self.state.get('project')}, the voyage for {self.voyage.project}")
|
||||
return EXIT_CONFIG
|
||||
gates = "decided by the system" if self.voyage.project in self.settings.gate_projects else "left to a person"
|
||||
self.log(f"Voyage {self.voyage.name} ({self.voyage.engagement_id}) in project {self.voyage.project}; gates {gates}.")
|
||||
self.log(f"Ledger: {self.ledger.folder}")
|
||||
try:
|
||||
for stage in self.voyage.stages:
|
||||
if only and stage.key not in only and self._stage_state(stage.key).get("state") != "completed":
|
||||
if any(s.key in only for s in self.voyage.stages[self.voyage.stages.index(stage) + 1:]):
|
||||
raise Stop(EXIT_CONFIG, f"stage {stage.key} isn't completed, and a later stage was asked for")
|
||||
continue
|
||||
self._stage(stage, dry_run)
|
||||
except Stop as stop:
|
||||
self.state["stopped"] = {"code": stop.code, "reason": stop.reason, "at": utc_iso(self.clock())}
|
||||
self._save()
|
||||
self.log(f"Stopped: {stop.reason}")
|
||||
return stop.code
|
||||
except (KabError, VoyageError) as error:
|
||||
self.state["stopped"] = {"code": EXIT_CONFIG, "reason": str(error), "at": utc_iso(self.clock())}
|
||||
self._save()
|
||||
self.log(f"Stopped: {error}")
|
||||
return EXIT_CONFIG
|
||||
self.state.pop("stopped", None)
|
||||
self.state["finished_at"] = utc_iso(self.clock())
|
||||
self._save()
|
||||
self.log("Dry run: nothing started." if dry_run else "Voyage complete.")
|
||||
return EXIT_OK
|
||||
|
||||
def _stage(self, stage: Stage, dry_run: bool) -> None:
|
||||
st = self._stage_state(stage.key)
|
||||
if st.get("state") == "completed":
|
||||
self.log(f"[{stage.key}] already completed: {st.get('execution_id')}")
|
||||
return
|
||||
wf = load_workflow(stage.workflow)
|
||||
workflow_id = stage.workflow_id or wf.workflow_id
|
||||
if stage.use_execution:
|
||||
st.update(execution_id=stage.use_execution, adopted=True, approval_ref=stage.approval_ref,
|
||||
workflow=wf.key, state="started")
|
||||
self._save()
|
||||
if not st.get("execution_id"):
|
||||
if not workflow_id:
|
||||
raise VoyageError(f"stage {stage.key}: no workflow_id in the voyage or workflows/{wf.key}.yaml")
|
||||
template = self._checked_template(stage, wf, workflow_id)
|
||||
inputs, context_text = self._inputs(stage, wf, workflow_id)
|
||||
st.update(workflow=wf.key, workflow_id=workflow_id, workflow_name=template.get("name"),
|
||||
workflow_version=template.get("version"), run_id=inputs.get("run_id"))
|
||||
self.ledger.write("inputs.json", shown_inputs(inputs), stage.key)
|
||||
if context_text is not None:
|
||||
self.ledger.write("context.yaml", context_text, stage.key)
|
||||
if dry_run:
|
||||
st["state"] = "planned"
|
||||
self._save()
|
||||
self.log(f"[{stage.key}] would start {template.get('name')} ({workflow_id}) with "
|
||||
f"{shown_inputs(inputs)}")
|
||||
return
|
||||
name = self.kab.project_name(self.voyage.project)
|
||||
body = cp1_agents.run_workflow().with_run_metadata(template, workflow_id, self.voyage.project, name)
|
||||
result = self.kab.execute(body, inputs)
|
||||
if not result.get("execution_id"):
|
||||
raise Stop(EXIT_STOPPED, f"stage {stage.key}: KAB started no execution: {str(result)[:300]}")
|
||||
st.update(execution_id=result["execution_id"], state="started", started_at=utc_iso(self.clock()))
|
||||
self._save()
|
||||
self.log(f"[{stage.key}] started {result['execution_id']} ({template.get('name')})")
|
||||
self._follow(stage, st)
|
||||
self._finish(stage, wf, st)
|
||||
|
||||
# ---- start ---------------------------------------------------------------------------------------------------
|
||||
def _checked_template(self, stage: Stage, wf: Workflow, workflow_id: str) -> dict:
|
||||
workflow = self.kab.workflow(workflow_id)
|
||||
template = workflow.get("template") or workflow
|
||||
if template.get("name") not in wf.names:
|
||||
raise VoyageError(f"stage {stage.key}: workflow {workflow_id} is {template.get('name')!r}, not one of "
|
||||
f"{wf.names} (ids change on re-import: update the voyage)")
|
||||
return template
|
||||
|
||||
def _resolve(self, stage: Stage, value: Any) -> Any:
|
||||
if not isinstance(value, dict):
|
||||
return value
|
||||
upstream = self.state["stages"].get(value["from"]) or {}
|
||||
if upstream.get("state") == "planned": # a dry run: the upstream run doesn't exist yet
|
||||
deferred = f"(at run time: {value['from']}.{value['value']})"
|
||||
return [deferred] if value["value"] == "package_artifact_ids" else deferred
|
||||
if upstream.get("state") != "completed":
|
||||
raise VoyageError(f"stage {stage.key} needs stage {value['from']} completed first")
|
||||
found = upstream.get(value["value"])
|
||||
if found in (None, "", []):
|
||||
raise VoyageError(f"stage {stage.key}: stage {value['from']} has no {value['value']}")
|
||||
return found
|
||||
|
||||
def _inputs(self, stage: Stage, wf: Workflow, workflow_id: str) -> tuple[dict, Optional[str]]:
|
||||
helpers = cp1_agents.run_workflow()
|
||||
given = {name: self._resolve(stage, value) for name, value in stage.inputs.items()}
|
||||
inputs: Dict[str, Any] = {}
|
||||
context_doc, context_text = None, None
|
||||
if stage.context:
|
||||
path = resolve_path(stage.context)
|
||||
context_doc = yaml.safe_load(path.read_text()) or {}
|
||||
from_file = (context_doc.get("workflow_inputs") or {}) if isinstance(context_doc, dict) else {}
|
||||
for key, value in from_file.items():
|
||||
value = "yes" if value is True else "no" if value is False else value
|
||||
if key in given and not (isinstance(value, str) and PLACEHOLDER.match(value)) and str(value) != str(given[key]):
|
||||
raise VoyageError(f"stage {stage.key}: {key} is {value!r} in {path.name} and {given[key]!r} in "
|
||||
"the voyage: keep one")
|
||||
inputs[key] = value if not isinstance(value, (int, float)) else str(value)
|
||||
inputs.update(given)
|
||||
if "run_id" in inputs:
|
||||
raise VoyageError(f"stage {stage.key}: run_id is generated for every run; remove it")
|
||||
if inputs.get("engagement_id") not in (None, self.voyage.engagement_id):
|
||||
raise VoyageError(f"stage {stage.key}: engagement_id {inputs.get('engagement_id')} is not the voyage's "
|
||||
f"{self.voyage.engagement_id}")
|
||||
inputs["engagement_id"] = self.voyage.engagement_id
|
||||
generated = helpers.new_ids()
|
||||
inputs["run_id"] = generated["run_id"]
|
||||
for name, doc_path in stage.documents.items():
|
||||
inputs[name] = resolve_path(doc_path).read_text()
|
||||
|
||||
if context_doc is not None:
|
||||
has_upstream = any(inputs.get(n) for n in helpers.UPSTREAM_INPUTS)
|
||||
problem = helpers.customer_problem(context_doc, has_upstream=has_upstream)
|
||||
if problem:
|
||||
raise VoyageError(f"stage {stage.key}: {stage.context}: {problem}")
|
||||
header = ("# Added by the Shakedown runner when the run started. Never edit these by hand.\n"
|
||||
"generated:\n" + "".join(f" {k}: {v}\n" for k, v in generated.items()) + "\n")
|
||||
path = resolve_path(stage.context)
|
||||
if wf.context == "upload":
|
||||
context_text = header + path.read_text()
|
||||
files = [{"name": path.name, "content": self._data_url(path, context_text.encode())}]
|
||||
for extra in helpers.context_uploads(path):
|
||||
files.append({"name": extra.name, "content": self._data_url(extra, extra.read_bytes())})
|
||||
inputs["file"] = files
|
||||
elif wf.context == "document":
|
||||
resolved = dict(context_doc)
|
||||
resolved["workflow_inputs"] = {k: inputs[k] for k in (context_doc.get("workflow_inputs") or {})
|
||||
if k in inputs}
|
||||
context_text = header + yaml.safe_dump(resolved, sort_keys=False, allow_unicode=True)
|
||||
inputs[wf.document_input] = context_text
|
||||
|
||||
leftover = sorted(k for k, v in inputs.items() if isinstance(v, str) and PLACEHOLDER.match(v.strip()))
|
||||
if leftover:
|
||||
raise VoyageError(f"stage {stage.key}: still a placeholder: {', '.join(leftover)}")
|
||||
sample = self.kab.api_sample(workflow_id)
|
||||
missing = sorted(f"{i['step_id']}.{i['name']}" for i in sample.get("required", [])
|
||||
if i["name"] != "file" and i["name"] not in inputs)
|
||||
if missing:
|
||||
raise VoyageError(f"stage {stage.key}: missing inputs {', '.join(missing)}")
|
||||
checkable = {k: v for k, v in inputs.items() if not (isinstance(v, str) and v.startswith("(at run time: "))}
|
||||
try:
|
||||
helpers.check_engagement(self.kab, checkable)
|
||||
except SystemExit as refused:
|
||||
raise VoyageError(f"stage {stage.key}: {refused}") from None
|
||||
return inputs, context_text
|
||||
|
||||
@staticmethod
|
||||
def _data_url(path: Path, data: bytes) -> str:
|
||||
kind = cp1_agents.run_workflow().upload_type(path)
|
||||
return f"data:{kind};base64,{base64.b64encode(data).decode()}"
|
||||
|
||||
# ---- follow --------------------------------------------------------------------------------------------------
|
||||
def _follow(self, stage: Stage, st: dict) -> None:
|
||||
eid = st["execution_id"]
|
||||
deadline = self.clock() + self.settings.stage_timeout_s
|
||||
while True:
|
||||
status = self.kab.status(eid)
|
||||
state = str(status.get("status", "")).lower()
|
||||
if state in DONE_OK or state in DONE_BAD:
|
||||
if st.get("state") == "needs_person": # a person decided and the run finished between two polls
|
||||
self._person_decided(stage, st)
|
||||
st["run_status"] = state
|
||||
self._save()
|
||||
return
|
||||
pending = (self.kab.reviews(eid) or {}).get("pending") or []
|
||||
if pending:
|
||||
self._at_gate(stage, st, status, pending)
|
||||
elif st.get("state") == "needs_person":
|
||||
self._person_decided(stage, st)
|
||||
if self.clock() > deadline:
|
||||
raise Stop(EXIT_TIMEOUT, f"stage {stage.key}: {eid} still {state} after "
|
||||
f"{self.settings.stage_timeout_s // 60} minutes")
|
||||
self.sleep(self.settings.poll_seconds)
|
||||
|
||||
def _at_gate(self, stage: Stage, st: dict, status: dict, pending: list) -> None:
|
||||
eid = st["execution_id"]
|
||||
review = pending[0]
|
||||
checkpoint = str(review.get("checkpoint_id"))
|
||||
if any(d.get("checkpoint_id") == checkpoint and d.get("decision") == "continue" for d in st["decisions"]):
|
||||
return # decided; KAB hasn't moved on yet
|
||||
system_may_decide = self.voyage.project in self.settings.gate_projects and len(pending) == 1
|
||||
if not system_may_decide:
|
||||
if st.get("state") != "needs_person":
|
||||
st.update(state="needs_person", waiting_since=self.clock(), checkpoint_id=checkpoint)
|
||||
self._save()
|
||||
why = "more than one pause" if len(pending) > 1 else "this project isn't one of the runner's gate_projects"
|
||||
self.log(f"[{stage.key}] {eid} waits at a review point for a person in KAB ({why}).")
|
||||
elif self.clock() - st.get("waiting_since", self.clock()) > self.settings.person_wait_s:
|
||||
raise Stop(EXIT_PERSON_TIMEOUT, f"stage {stage.key}: nobody decided {eid}'s review point in "
|
||||
f"{self.settings.person_wait_s // 3600} hours")
|
||||
return
|
||||
steps = self.kab.steps(eid)
|
||||
verdict = gate.decide(status, steps, review.get("step_id"))
|
||||
at = utc_iso(self.clock())
|
||||
record = {"checkpoint_id": checkpoint, "step_id": review.get("step_id"), "at": at, "by": "system",
|
||||
"verdict": verdict.verdict, "verdict_step": verdict.verdict_step, "reason": verdict.reason}
|
||||
if not verdict.proceed:
|
||||
record["decision"] = "left for a fix"
|
||||
st["decisions"].append(record)
|
||||
st["state"] = "needs_fix"
|
||||
self.ledger.write("steps-at-gate.json", steps, stage.key)
|
||||
self._save()
|
||||
raise Stop(EXIT_STOPPED, f"stage {stage.key}: not continued at the review point: {verdict.reason}. "
|
||||
f"{eid} stays paused for a person or a fix.")
|
||||
note = (f"Decided by the system (Shakedown runner, KAB automation identity), not a person: continue. "
|
||||
f"{verdict.reason}. Execution {eid}, run {st.get('run_id') or '?'}, voyage {self.voyage.name}, {at}.")
|
||||
self.kab.decide(eid, checkpoint, "continue", note)
|
||||
record.update(decision="continue", note=note)
|
||||
st["decisions"].append(record)
|
||||
st.update(state="started", approval_ref=f"Shakedown runner (system) continued {eid} at {at}: "
|
||||
f"critic verdict pass at {verdict.verdict_step}")
|
||||
self._save()
|
||||
self.log(f"[{stage.key}] continued {eid}: {verdict.reason}")
|
||||
|
||||
def _person_decided(self, stage: Stage, st: dict) -> None:
|
||||
"""The pause is gone and the run moves on: a person decided in KAB. Record who, as best KAB says."""
|
||||
eid = st["execution_id"]
|
||||
reviews = self.kab.reviews(eid) or {}
|
||||
decided = [r for key in ("history", "decided", "completed", "reviews") for r in (reviews.get(key) or [])
|
||||
if isinstance(r, dict) and str(r.get("checkpoint_id")) == str(st.get("checkpoint_id"))]
|
||||
last = decided[-1] if decided else {}
|
||||
who = last.get("decided_by") or last.get("reviewer") or last.get("user") or "a person in KAB"
|
||||
when = last.get("decided_at") or last.get("updated_at") or utc_iso(self.clock())
|
||||
decision = last.get("decision") or "continue"
|
||||
st["decisions"].append({"checkpoint_id": st.get("checkpoint_id"), "by": "person", "decision": decision,
|
||||
"who": who, "at": when})
|
||||
st.update(state="started", approval_ref=f"{who} decided {decision} on {eid} at {when}")
|
||||
st.pop("waiting_since", None)
|
||||
self._save()
|
||||
self.log(f"[{stage.key}] {who} decided {decision} on {eid}")
|
||||
|
||||
# ---- finish --------------------------------------------------------------------------------------------------
|
||||
def _finish(self, stage: Stage, wf: Workflow, st: dict) -> None:
|
||||
eid = st["execution_id"]
|
||||
status = self.kab.status(eid)
|
||||
steps = self.kab.steps(eid)
|
||||
self.ledger.write("status.json", status, stage.key)
|
||||
self.ledger.write("steps.json", steps, stage.key)
|
||||
run_status = str(status.get("status", "")).lower()
|
||||
if run_status not in DONE_OK:
|
||||
st["state"] = "failed"
|
||||
self._save()
|
||||
raise Stop(EXIT_STOPPED, f"stage {stage.key}: {eid} ended {run_status or 'with no status'}")
|
||||
artifact = self._package_artifact(eid, status, steps, st.get("workflow_id"))
|
||||
if artifact is None:
|
||||
st["state"] = "failed"
|
||||
self._save()
|
||||
raise Stop(EXIT_STOPPED, f"stage {stage.key}: {eid} completed with no package (no output has a manifest "
|
||||
"with a package_version)")
|
||||
st["package_artifact_ids"] = [artifact]
|
||||
if not st.get("approval_ref"):
|
||||
st["approval_ref"] = f"{eid} completed with no review point decided by the runner"
|
||||
if stage.case:
|
||||
case = yaml.safe_load(resolve_path(stage.case).read_text())
|
||||
rows, failed = self.evaluator(wf.feature, case, steps)
|
||||
evaluation = {"case": stage.case, "failed": failed,
|
||||
"checks": [dict(zip(("status", "severity", "where", "id", "detail"), r)) for r in rows]}
|
||||
self.ledger.write("evaluation.json", evaluation, stage.key)
|
||||
st["evaluation"] = {"failed": failed, "checks": len(rows)}
|
||||
if failed.get("critical"):
|
||||
st["state"] = "checks_failed"
|
||||
self._save()
|
||||
raise Stop(EXIT_STOPPED, f"stage {stage.key}: {failed['critical']} critical check(s) failed on {eid} "
|
||||
f"(see {stage.key}/evaluation.json)")
|
||||
st.update(state="completed", completed_at=utc_iso(self.clock()))
|
||||
self._save()
|
||||
self.log(f"[{stage.key}] completed {eid}; package {artifact}")
|
||||
|
||||
def _package_artifact(self, eid: str, status: dict, steps: Any, workflow_id: Optional[str]) -> Optional[str]:
|
||||
"""`<execution id>_step_<n>`, n the package step's 1-based place in the executed template."""
|
||||
outputs = gate.step_outputs(steps)
|
||||
holder = next((k for k, v in outputs.items() if isinstance(v, dict) and isinstance(v.get("manifest"), dict)
|
||||
and "package_version" in v["manifest"]), None)
|
||||
if holder is None:
|
||||
return None
|
||||
order = gate.template_order(status)
|
||||
if holder not in order and workflow_id:
|
||||
workflow = self.kab.workflow(workflow_id)
|
||||
order = [str(s.get("id")) for s in ((workflow.get("template") or workflow).get("steps") or [])]
|
||||
if holder not in order:
|
||||
return None
|
||||
return f"{eid}_step_{order.index(holder) + 1}"
|
||||
Reference in New Issue
Block a user