Runner: run a voyage remotely as the kab-automation identity
- shakedown_lib: runs F01 -> F02 -> F03 for one engagement on KAB with no personal token. Signs in as the kab-automation client (client credentials, provisioned by Stellwerk from crucible-platform-deploy) and renews the token itself. - Each stage checks its workflow id against the registered sub-agent name, builds inputs from the case and upstream stages, starts the run, follows it and judges it with cp1-agents evaluate_case.py. - Gates (plan 5b): in config/runner.yaml gate_projects only, continue on a critic pass with no failed step and a note saying the system decided. Never on a fail. Elsewhere, wait for a person and record who decided. gate_projects is empty for now. - Ledger per voyage run, with --resume; nothing secret is written. - Cymbal voyage ENG-CYMBAL-001; workflow registry; Dockerfile and an unapplied Job template; 15 tests against a fake KAB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
0
tests/__init__.py
Normal file
0
tests/__init__.py
Normal file
320
tests/test_runner.py
Normal file
320
tests/test_runner.py
Normal file
@@ -0,0 +1,320 @@
|
||||
"""The runner against a fake KAB: python3 -m unittest discover -s tests (needs the cp1-agents clone beside this repo,
|
||||
or CP1_AGENTS_DIR)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yaml
|
||||
|
||||
from shakedown_lib import gate
|
||||
from shakedown_lib.auth import AuthError, ClientCredentials
|
||||
from shakedown_lib.kab import Kab
|
||||
from shakedown_lib.ledger import Ledger
|
||||
from shakedown_lib.runner import EXIT_CONFIG, EXIT_OK, EXIT_STOPPED, Runner, Settings
|
||||
from shakedown_lib.voyage import load_voyage
|
||||
|
||||
PROJECT = "proj-1"
|
||||
ENG = "ENG-TEST-001"
|
||||
NAMES = {"wf-f01": "CP1 - Agentic Landing Zone - Discovery", "wf-f02": "CP1 - Agentic Landing Zone - Assessment",
|
||||
"wf-f03": "CP1 - Agentic Landing Zone - Strategy Synthesis"}
|
||||
|
||||
|
||||
def template(name: str, upload: bool) -> dict:
|
||||
first = "u1" if upload else "a1"
|
||||
return {"name": name, "version": "1.0", "steps": [{"id": first}] + [{"id": f"s{i}"} for i in range(1, 7)] + [{"id": "f1"}]}
|
||||
|
||||
|
||||
class FakeKab:
|
||||
"""Runs move: running -> paused at s5 (one pending review) -> after `continue`, completed with a package at s6."""
|
||||
|
||||
def __init__(self, verdict: str = "pass", polls_before_pause: int = 1, person_after: int | None = None,
|
||||
names: dict | None = None) -> None:
|
||||
self.verdict = verdict
|
||||
self.polls_before_pause = polls_before_pause
|
||||
self.person_after = person_after
|
||||
self.names = names or NAMES
|
||||
self.runs: dict = {"exec_f01_done": {"workflow": "wf-f01", "inputs": {"engagement_id": ENG}, "phase": "done",
|
||||
"polls": 0, "history": []}}
|
||||
self.executed: list = []
|
||||
self.decisions: list = []
|
||||
|
||||
def transport(self, method, url, headers, data):
|
||||
assert headers["Authorization"].startswith("Bearer "), "no token sent"
|
||||
path = urlparse(url).path
|
||||
body = json.loads(data) if data else None
|
||||
reply = self.route(method, path, body)
|
||||
return 200, json.dumps(reply).encode()
|
||||
|
||||
def _status(self, eid):
|
||||
run = self.runs[eid]
|
||||
state = {"running": "running", "paused": "attention", "done": "completed"}[run["phase"]]
|
||||
return {"execution_id": eid, "status": state, "inputs": run["inputs"],
|
||||
"template": template(self.names[run["workflow"]], run["workflow"] == "wf-f01")}
|
||||
|
||||
def _steps(self, eid):
|
||||
run = self.runs[eid]
|
||||
steps = {"s4": {"response": json.dumps({"phase": "critique", "status": "completed",
|
||||
"verdict": {"verdict": self.verdict}})}}
|
||||
if run["phase"] == "done":
|
||||
steps["s6"] = {"output": {"engagement_id": ENG, "manifest": {"package_version": "1.0"}}}
|
||||
return {"execution_id": eid, "steps": steps}
|
||||
|
||||
def route(self, method, path, body):
|
||||
if path.startswith("/api/v1/workflow-catalog/workflows/"):
|
||||
wid = path.rsplit("/", 1)[1]
|
||||
return {"workflow": {"template": template(self.names[wid], wid == "wf-f01")}}
|
||||
if path.endswith("/api-sample"):
|
||||
return {"required": []}
|
||||
if path.startswith("/api/v1/projects/"):
|
||||
return {"project": {"name": "Test project"}}
|
||||
if path == "/api/workflows/execute":
|
||||
wid = body["template"]["metadata"]["workflow_id"]
|
||||
eid = f"exec_{len(self.executed) + 1}"
|
||||
self.executed.append({"workflow": wid, "inputs": body["inputs"], "template": body["template"]})
|
||||
self.runs[eid] = {"workflow": wid, "inputs": body["inputs"], "phase": "running", "polls": 0, "history": []}
|
||||
return {"execution_id": eid, "status": "running"}
|
||||
if path.startswith("/api/workflows/status/"):
|
||||
eid = path.rsplit("/", 1)[1]
|
||||
run = self.runs[eid]
|
||||
run["polls"] += 1
|
||||
if run["phase"] == "running" and run["polls"] > self.polls_before_pause:
|
||||
run["phase"] = "paused"
|
||||
run["paused_at"] = run["polls"]
|
||||
if run["phase"] == "paused" and self.person_after is not None and run["polls"] - run["paused_at"] >= self.person_after:
|
||||
run["phase"] = "done"
|
||||
run["history"].append({"checkpoint_id": f"cp-{eid}", "decision": "continue",
|
||||
"decided_by": "lead@example.com", "decided_at": "2026-11-16T10:00:00Z"})
|
||||
return self._status(eid)
|
||||
if path.endswith("/steps"):
|
||||
return self._steps(path.split("/")[-2])
|
||||
if path.endswith("/reviews") and method == "GET":
|
||||
eid = path.split("/")[-2]
|
||||
run = self.runs[eid]
|
||||
pending = [{"checkpoint_id": f"cp-{eid}", "step_id": "s5"}] if run["phase"] == "paused" else []
|
||||
return {"pending": pending, "history": run["history"]}
|
||||
if "/reviews/" in path and method == "POST":
|
||||
eid = path.split("/")[-3]
|
||||
self.decisions.append({"execution": eid, **body})
|
||||
if body["decision"] == "continue":
|
||||
self.runs[eid]["phase"] = "done"
|
||||
return {"ok": True}
|
||||
raise AssertionError(f"unexpected {method} {path}")
|
||||
|
||||
|
||||
class Tokens:
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
|
||||
def token(self, force=False):
|
||||
self.calls += 1
|
||||
return "t"
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self):
|
||||
self.now = 1_800_000_000.0
|
||||
|
||||
def __call__(self):
|
||||
return self.now
|
||||
|
||||
def sleep(self, seconds):
|
||||
self.now += seconds
|
||||
|
||||
|
||||
def write_voyage(folder: Path, f01_adopted: bool = True) -> Path:
|
||||
(folder / "context.yaml").write_text(textwrap.dedent(f"""\
|
||||
workflow_inputs:
|
||||
engagement_id: {ENG}
|
||||
provider: sample
|
||||
include_cmdb: yes
|
||||
customer: {{id: CUS-TEST, name: Test}}
|
||||
"""))
|
||||
(folder / "run-context.yaml").write_text(textwrap.dedent(f"""\
|
||||
workflow_inputs:
|
||||
engagement_id: {ENG}
|
||||
assessment_execution_id: <assessment execution id>
|
||||
assessment_approval_ref: "<who continued it>"
|
||||
context:
|
||||
customer: {{id: CUS-TEST}}
|
||||
"""))
|
||||
(folder / "rate-card.json").write_text('{"rates": []}')
|
||||
(folder / "case.yaml").write_text("id: test\nchecks: []\n")
|
||||
f01 = {"key": "f01", "workflow": "alz-f01-discovery", "workflow_id": "wf-f01", "context": f"{folder}/context.yaml"}
|
||||
if f01_adopted:
|
||||
f01.update(use_execution="exec_f01_done", approval_ref="approved by a person")
|
||||
doc = {"voyage": ENG, "engagement_id": ENG, "project": PROJECT, "stages": [
|
||||
f01,
|
||||
{"key": "f02", "workflow": "alz-f02-assessment", "workflow_id": "wf-f02",
|
||||
"inputs": {"discovery_execution_id": {"from": "f01", "value": "execution_id"},
|
||||
"discovery_approval_ref": {"from": "f01", "value": "approval_ref"},
|
||||
"artifact_ids": {"from": "f01", "value": "package_artifact_ids"}},
|
||||
"documents": {"rate_card_document": f"{folder}/rate-card.json"}, "case": f"{folder}/case.yaml"},
|
||||
{"key": "f03", "workflow": "alz-f03-strategy", "workflow_id": "wf-f03", "context": f"{folder}/run-context.yaml",
|
||||
"inputs": {"assessment_execution_id": {"from": "f02", "value": "execution_id"},
|
||||
"assessment_approval_ref": {"from": "f02", "value": "approval_ref"},
|
||||
"artifact_ids": {"from": "f02", "value": "package_artifact_ids"}}}]}
|
||||
voyage = folder / "voyage.yaml"
|
||||
voyage.write_text(yaml.safe_dump(doc, sort_keys=False))
|
||||
return voyage
|
||||
|
||||
|
||||
class RunnerTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.dir = Path(self.tmp.name)
|
||||
self.clock = Clock()
|
||||
self.evaluated = []
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def runner(self, fake: FakeKab, gate_projects=(PROJECT,), ledger: Ledger | None = None, adopted=True, **settings):
|
||||
voyage = load_voyage(write_voyage(self.dir, adopted))
|
||||
kab = Kab(Tokens(), api="https://kab.test", tenant="t", project=PROJECT, transport=fake.transport,
|
||||
sleep=lambda s: None)
|
||||
|
||||
def evaluator(feature, case, steps):
|
||||
self.evaluated.append(feature)
|
||||
return [("pass", "critical", "manifest", "c1", "")], {"critical": 0, "major": 0}
|
||||
|
||||
return Runner(kab=kab, ledger=ledger or Ledger(self.dir / "ledger"), voyage=voyage,
|
||||
settings=Settings(gate_projects=set(gate_projects), poll_seconds=30, **settings),
|
||||
evaluator=evaluator, clock=self.clock, sleep=self.clock.sleep, log=lambda m: None)
|
||||
|
||||
def test_whole_voyage_decides_gates_in_its_own_project(self):
|
||||
fake = FakeKab()
|
||||
runner = self.runner(fake)
|
||||
self.assertEqual(runner.run(), EXIT_OK)
|
||||
self.assertEqual([e["workflow"] for e in fake.executed], ["wf-f02", "wf-f03"])
|
||||
f02, f03 = fake.executed
|
||||
self.assertEqual(f02["inputs"]["discovery_execution_id"], "exec_f01_done")
|
||||
self.assertEqual(f02["inputs"]["artifact_ids"], ["exec_f01_done_step_7"])
|
||||
self.assertEqual(f02["inputs"]["rate_card_document"], '{"rates": []}')
|
||||
self.assertEqual(f02["template"]["metadata"]["project_id"], PROJECT)
|
||||
self.assertEqual(f03["inputs"]["artifact_ids"], ["exec_1_step_7"])
|
||||
self.assertIn("Shakedown runner (system) continued exec_1", f03["inputs"]["assessment_approval_ref"])
|
||||
document = f03["inputs"]["run_context_document"]
|
||||
self.assertIn("assessment_execution_id: exec_1", document)
|
||||
self.assertNotIn("<assessment execution id>", document)
|
||||
self.assertEqual([d["decision"] for d in fake.decisions], ["continue", "continue"])
|
||||
self.assertTrue(all("not a person" in d["note"] for d in fake.decisions))
|
||||
state = json.loads((self.dir / "ledger" / "state.json").read_text())
|
||||
self.assertEqual({k: v["state"] for k, v in state["stages"].items()},
|
||||
{"f01": "completed", "f02": "completed", "f03": "completed"})
|
||||
self.assertEqual(self.evaluated, ["f02"])
|
||||
self.assertTrue((self.dir / "ledger" / "f03" / "context.yaml").is_file())
|
||||
|
||||
def test_critic_fail_is_never_continued(self):
|
||||
fake = FakeKab(verdict="fail")
|
||||
self.assertEqual(self.runner(fake).run(), EXIT_STOPPED)
|
||||
self.assertEqual(fake.decisions, [])
|
||||
state = json.loads((self.dir / "ledger" / "state.json").read_text())
|
||||
self.assertEqual(state["stages"]["f02"]["state"], "needs_fix")
|
||||
self.assertIn("critic verdict is 'fail'", state["stopped"]["reason"])
|
||||
|
||||
def test_other_projects_wait_for_a_person(self):
|
||||
fake = FakeKab(person_after=3)
|
||||
self.assertEqual(self.runner(fake, gate_projects=()).run(), EXIT_OK)
|
||||
self.assertEqual(fake.decisions, [])
|
||||
f03 = fake.executed[1]
|
||||
self.assertEqual(f03["inputs"]["assessment_approval_ref"],
|
||||
"lead@example.com decided continue on exec_1 at 2026-11-16T10:00:00Z")
|
||||
|
||||
def test_nobody_deciding_times_out(self):
|
||||
fake = FakeKab()
|
||||
code = self.runner(fake, gate_projects=(), person_wait_s=120).run()
|
||||
self.assertEqual(code, 4)
|
||||
self.assertEqual(fake.decisions, [])
|
||||
|
||||
def test_a_stale_workflow_id_is_refused(self):
|
||||
fake = FakeKab(names={**NAMES, "wf-f02": "CP1 - Agentic Landing Zone - Assessment (individual agents)"})
|
||||
self.assertEqual(self.runner(fake).run(), EXIT_CONFIG)
|
||||
self.assertEqual(fake.executed, [])
|
||||
|
||||
def test_resume_never_starts_a_run_twice(self):
|
||||
fake = FakeKab(verdict="fail")
|
||||
ledger = Ledger(self.dir / "ledger")
|
||||
self.assertEqual(self.runner(fake, ledger=ledger).run(), EXIT_STOPPED)
|
||||
fake.verdict = "pass" # say a fix was deployed and the stage rerun in KAB
|
||||
self.assertEqual(self.runner(fake, ledger=ledger).run(), EXIT_OK)
|
||||
self.assertEqual([e["workflow"] for e in fake.executed], ["wf-f02", "wf-f03"])
|
||||
|
||||
def test_dry_run_starts_nothing(self):
|
||||
fake = FakeKab()
|
||||
self.assertEqual(self.runner(fake, adopted=False).run(dry_run=True), EXIT_OK)
|
||||
self.assertEqual(fake.executed, [])
|
||||
shown = json.loads((self.dir / "ledger" / "f02" / "inputs.json").read_text())
|
||||
self.assertEqual(shown["discovery_execution_id"], "(at run time: f01.execution_id)")
|
||||
f01 = json.loads((self.dir / "ledger" / "f01" / "inputs.json").read_text())
|
||||
self.assertEqual(f01["file"], ["context.yaml"])
|
||||
self.assertEqual(f01["include_cmdb"], "yes")
|
||||
|
||||
|
||||
class GateTest(unittest.TestCase):
|
||||
def status(self):
|
||||
return {"status": "attention", "template": {"steps": [{"id": i} for i in ("a1", "s1", "s2", "s3", "s4", "s5", "s6")]}}
|
||||
|
||||
def test_pass(self):
|
||||
steps = {"steps": {"s4": {"phase": "critique", "verdict": {"verdict": "pass"}}}}
|
||||
self.assertTrue(gate.decide(self.status(), steps, "s5").proceed)
|
||||
|
||||
def test_a_later_fail_overrides_an_earlier_pass(self):
|
||||
steps = {"steps": {"s3": {"phase": "x", "verdict": "pass"}, "s4": {"phase": "critique", "verdict": {"verdict": "fail"}}}}
|
||||
self.assertFalse(gate.decide(self.status(), steps, "s5").proceed)
|
||||
|
||||
def test_no_verdict(self):
|
||||
self.assertFalse(gate.decide(self.status(), {"steps": {"s1": {"phase": "plan"}}}, "s5").proceed)
|
||||
|
||||
def test_a_failed_step(self):
|
||||
steps = {"steps": {"s2": {"phase": "collect", "status": "failed"}, "s4": {"phase": "c", "verdict": "pass"}}}
|
||||
self.assertFalse(gate.decide(self.status(), steps, "s5").proceed)
|
||||
|
||||
def test_unparseable_verdict(self):
|
||||
steps = {"steps": {"s4": {"phase": "critique", "verdict": {"verdict": None}}}}
|
||||
self.assertFalse(gate.decide(self.status(), steps, "s5").proceed)
|
||||
|
||||
|
||||
class AuthTest(unittest.TestCase):
|
||||
def test_renews_before_expiry_and_hides_the_secret(self):
|
||||
clock = Clock()
|
||||
issued = []
|
||||
|
||||
def post(url, form):
|
||||
issued.append(form["grant_type"])
|
||||
return {"access_token": f"tok{len(issued)}", "expires_in": 300}
|
||||
|
||||
tokens = ClientCredentials("kab-automation", "s3cret", post=post, clock=clock)
|
||||
self.assertEqual(tokens.token(), "tok1")
|
||||
clock.now += 200
|
||||
self.assertEqual(tokens.token(), "tok1")
|
||||
clock.now += 50 # inside the last minute
|
||||
self.assertEqual(tokens.token(), "tok2")
|
||||
self.assertEqual(issued, ["client_credentials", "client_credentials"])
|
||||
self.assertNotIn("s3cret", repr(tokens))
|
||||
|
||||
def test_no_secret(self):
|
||||
with self.assertRaises(AuthError):
|
||||
ClientCredentials("kab-automation", "")
|
||||
|
||||
def test_a_401_renews_the_token_once(self):
|
||||
calls = []
|
||||
|
||||
class T:
|
||||
def token(self, force=False):
|
||||
calls.append(force)
|
||||
return "x"
|
||||
|
||||
replies = iter([(401, b""), (200, b'{"ok": 1}')])
|
||||
kab = Kab(T(), transport=lambda *a: next(replies), sleep=lambda s: None)
|
||||
self.assertEqual(kab.call("GET", "/x"), {"ok": 1})
|
||||
self.assertEqual(calls, [False, True])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user