"""The runner against a fake KAB: python3 -m unittest discover -s tests (needs the cp1-agents clone beside this repo, or CP1_AGENTS_DIR).""" from __future__ import annotations import json import tempfile import textwrap import unittest from pathlib import Path from urllib.parse import urlparse import yaml from shakedown_lib import gate from shakedown_lib.auth import AuthError, ClientCredentials from shakedown_lib.kab import Kab from shakedown_lib.ledger import Ledger from shakedown_lib.runner import EXIT_CONFIG, EXIT_OK, EXIT_STOPPED, Runner, Settings from shakedown_lib.voyage import load_voyage PROJECT = "proj-1" ENG = "ENG-TEST-001" NAMES = {"wf-f01": "CP1 - Agentic Landing Zone - Discovery", "wf-f02": "CP1 - Agentic Landing Zone - Assessment", "wf-f03": "CP1 - Agentic Landing Zone - Strategy Synthesis"} def template(name: str, upload: bool) -> dict: first = "u1" if upload else "a1" return {"name": name, "version": "1.0", "steps": [{"id": first}] + [{"id": f"s{i}"} for i in range(1, 7)] + [{"id": "f1"}]} class FakeKab: """Runs move: running -> paused at s5 (one pending review) -> after `continue`, completed with a package at s6.""" def __init__(self, verdict: str = "pass", polls_before_pause: int = 1, person_after: int | None = None, names: dict | None = None) -> None: self.verdict = verdict self.polls_before_pause = polls_before_pause self.person_after = person_after self.names = names or NAMES self.runs: dict = {"exec_f01_done": {"workflow": "wf-f01", "inputs": {"engagement_id": ENG}, "phase": "done", "polls": 0, "history": []}} self.executed: list = [] self.decisions: list = [] def transport(self, method, url, headers, data): assert headers["Authorization"].startswith("Bearer "), "no token sent" path = urlparse(url).path body = json.loads(data) if data else None reply = self.route(method, path, body) return 200, json.dumps(reply).encode() def _status(self, eid): run = self.runs[eid] state = {"running": "running", "paused": "attention", "done": "completed"}[run["phase"]] return {"execution_id": eid, "status": state, "inputs": run["inputs"], "template": template(self.names[run["workflow"]], run["workflow"] == "wf-f01")} def _steps(self, eid): run = self.runs[eid] steps = {"s4": {"response": json.dumps({"phase": "critique", "status": "completed", "verdict": {"verdict": self.verdict}})}} if run["phase"] == "done": steps["s6"] = {"output": {"engagement_id": ENG, "manifest": {"package_version": "1.0"}}} return {"execution_id": eid, "steps": steps} def route(self, method, path, body): if path.startswith("/api/v1/workflow-catalog/workflows/"): wid = path.rsplit("/", 1)[1] return {"workflow": {"template": template(self.names[wid], wid == "wf-f01")}} if path.endswith("/api-sample"): return {"required": []} if path.startswith("/api/v1/projects/"): return {"project": {"name": "Test project"}} if path == "/api/workflows/execute": wid = body["template"]["metadata"]["workflow_id"] eid = f"exec_{len(self.executed) + 1}" self.executed.append({"workflow": wid, "inputs": body["inputs"], "template": body["template"]}) self.runs[eid] = {"workflow": wid, "inputs": body["inputs"], "phase": "running", "polls": 0, "history": []} return {"execution_id": eid, "status": "running"} if path.startswith("/api/workflows/status/"): eid = path.rsplit("/", 1)[1] run = self.runs[eid] run["polls"] += 1 if run["phase"] == "running" and run["polls"] > self.polls_before_pause: run["phase"] = "paused" run["paused_at"] = run["polls"] if run["phase"] == "paused" and self.person_after is not None and run["polls"] - run["paused_at"] >= self.person_after: run["phase"] = "done" run["history"].append({"checkpoint_id": f"cp-{eid}", "decision": "continue", "decided_by": "lead@example.com", "decided_at": "2026-11-16T10:00:00Z"}) return self._status(eid) if path.endswith("/steps"): return self._steps(path.split("/")[-2]) if path.endswith("/reviews") and method == "GET": eid = path.split("/")[-2] run = self.runs[eid] pending = [{"checkpoint_id": f"cp-{eid}", "step_id": "s5"}] if run["phase"] == "paused" else [] return {"pending": pending, "history": run["history"]} if "/reviews/" in path and method == "POST": eid = path.split("/")[-3] self.decisions.append({"execution": eid, **body}) if body["decision"] == "continue": self.runs[eid]["phase"] = "done" return {"ok": True} raise AssertionError(f"unexpected {method} {path}") class Tokens: def __init__(self): self.calls = 0 def token(self, force=False): self.calls += 1 return "t" class Clock: def __init__(self): self.now = 1_800_000_000.0 def __call__(self): return self.now def sleep(self, seconds): self.now += seconds def write_voyage(folder: Path, f01_adopted: bool = True) -> Path: (folder / "context.yaml").write_text(textwrap.dedent(f"""\ workflow_inputs: engagement_id: {ENG} provider: sample include_cmdb: yes customer: {{id: CUS-TEST, name: Test}} """)) (folder / "run-context.yaml").write_text(textwrap.dedent(f"""\ workflow_inputs: engagement_id: {ENG} assessment_execution_id: assessment_approval_ref: "" context: customer: {{id: CUS-TEST}} """)) (folder / "rate-card.json").write_text('{"rates": []}') (folder / "case.yaml").write_text("id: test\nchecks: []\n") f01 = {"key": "f01", "workflow": "alz-f01-discovery", "workflow_id": "wf-f01", "context": f"{folder}/context.yaml"} if f01_adopted: f01.update(use_execution="exec_f01_done", approval_ref="approved by a person") doc = {"voyage": ENG, "engagement_id": ENG, "project": PROJECT, "stages": [ f01, {"key": "f02", "workflow": "alz-f02-assessment", "workflow_id": "wf-f02", "inputs": {"discovery_execution_id": {"from": "f01", "value": "execution_id"}, "discovery_approval_ref": {"from": "f01", "value": "approval_ref"}, "artifact_ids": {"from": "f01", "value": "package_artifact_ids"}}, "documents": {"rate_card_document": f"{folder}/rate-card.json"}, "case": f"{folder}/case.yaml"}, {"key": "f03", "workflow": "alz-f03-strategy", "workflow_id": "wf-f03", "context": f"{folder}/run-context.yaml", "inputs": {"assessment_execution_id": {"from": "f02", "value": "execution_id"}, "assessment_approval_ref": {"from": "f02", "value": "approval_ref"}, "artifact_ids": {"from": "f02", "value": "package_artifact_ids"}}}]} voyage = folder / "voyage.yaml" voyage.write_text(yaml.safe_dump(doc, sort_keys=False)) return voyage class RunnerTest(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.dir = Path(self.tmp.name) self.clock = Clock() self.evaluated = [] def tearDown(self): self.tmp.cleanup() def runner(self, fake: FakeKab, gate_projects=(PROJECT,), ledger: Ledger | None = None, adopted=True, **settings): voyage = load_voyage(write_voyage(self.dir, adopted)) kab = Kab(Tokens(), api="https://kab.test", tenant="t", project=PROJECT, transport=fake.transport, sleep=lambda s: None) def evaluator(feature, case, steps): self.evaluated.append(feature) return [("pass", "critical", "manifest", "c1", "")], {"critical": 0, "major": 0} return Runner(kab=kab, ledger=ledger or Ledger(self.dir / "ledger"), voyage=voyage, settings=Settings(gate_projects=set(gate_projects), poll_seconds=30, **settings), evaluator=evaluator, clock=self.clock, sleep=self.clock.sleep, log=lambda m: None) def test_whole_voyage_decides_gates_in_its_own_project(self): fake = FakeKab() runner = self.runner(fake) self.assertEqual(runner.run(), EXIT_OK) self.assertEqual([e["workflow"] for e in fake.executed], ["wf-f02", "wf-f03"]) f02, f03 = fake.executed self.assertEqual(f02["inputs"]["discovery_execution_id"], "exec_f01_done") self.assertEqual(f02["inputs"]["artifact_ids"], ["exec_f01_done_step_7"]) self.assertEqual(f02["inputs"]["rate_card_document"], '{"rates": []}') self.assertEqual(f02["template"]["metadata"]["project_id"], PROJECT) self.assertEqual(f03["inputs"]["artifact_ids"], ["exec_1_step_7"]) self.assertIn("Shakedown runner (system) continued exec_1", f03["inputs"]["assessment_approval_ref"]) document = f03["inputs"]["run_context_document"] self.assertIn("assessment_execution_id: exec_1", document) self.assertNotIn("", document) self.assertEqual([d["decision"] for d in fake.decisions], ["continue", "continue"]) self.assertTrue(all("not a person" in d["note"] for d in fake.decisions)) state = json.loads((self.dir / "ledger" / "state.json").read_text()) self.assertEqual({k: v["state"] for k, v in state["stages"].items()}, {"f01": "completed", "f02": "completed", "f03": "completed"}) self.assertEqual(self.evaluated, ["f02"]) self.assertTrue((self.dir / "ledger" / "f03" / "context.yaml").is_file()) def test_critic_fail_is_never_continued(self): fake = FakeKab(verdict="fail") self.assertEqual(self.runner(fake).run(), EXIT_STOPPED) self.assertEqual(fake.decisions, []) state = json.loads((self.dir / "ledger" / "state.json").read_text()) self.assertEqual(state["stages"]["f02"]["state"], "needs_fix") self.assertIn("critic verdict is 'fail'", state["stopped"]["reason"]) def test_other_projects_wait_for_a_person(self): fake = FakeKab(person_after=3) self.assertEqual(self.runner(fake, gate_projects=()).run(), EXIT_OK) self.assertEqual(fake.decisions, []) f03 = fake.executed[1] self.assertEqual(f03["inputs"]["assessment_approval_ref"], "lead@example.com decided continue on exec_1 at 2026-11-16T10:00:00Z") def test_nobody_deciding_times_out(self): fake = FakeKab() code = self.runner(fake, gate_projects=(), person_wait_s=120).run() self.assertEqual(code, 4) self.assertEqual(fake.decisions, []) def test_a_stale_workflow_id_is_refused(self): fake = FakeKab(names={**NAMES, "wf-f02": "CP1 - Agentic Landing Zone - Assessment (individual agents)"}) self.assertEqual(self.runner(fake).run(), EXIT_CONFIG) self.assertEqual(fake.executed, []) def test_resume_never_starts_a_run_twice(self): fake = FakeKab(verdict="fail") ledger = Ledger(self.dir / "ledger") self.assertEqual(self.runner(fake, ledger=ledger).run(), EXIT_STOPPED) fake.verdict = "pass" # say a fix was deployed and the stage rerun in KAB self.assertEqual(self.runner(fake, ledger=ledger).run(), EXIT_OK) self.assertEqual([e["workflow"] for e in fake.executed], ["wf-f02", "wf-f03"]) def test_dry_run_starts_nothing(self): fake = FakeKab() self.assertEqual(self.runner(fake, adopted=False).run(dry_run=True), EXIT_OK) self.assertEqual(fake.executed, []) shown = json.loads((self.dir / "ledger" / "f02" / "inputs.json").read_text()) self.assertEqual(shown["discovery_execution_id"], "(at run time: f01.execution_id)") f01 = json.loads((self.dir / "ledger" / "f01" / "inputs.json").read_text()) self.assertEqual(f01["file"], ["context.yaml"]) self.assertEqual(f01["include_cmdb"], "yes") class GateTest(unittest.TestCase): def status(self): return {"status": "attention", "template": {"steps": [{"id": i} for i in ("a1", "s1", "s2", "s3", "s4", "s5", "s6")]}} def test_pass(self): steps = {"steps": {"s4": {"phase": "critique", "verdict": {"verdict": "pass"}}}} self.assertTrue(gate.decide(self.status(), steps, "s5").proceed) def test_a_later_fail_overrides_an_earlier_pass(self): steps = {"steps": {"s3": {"phase": "x", "verdict": "pass"}, "s4": {"phase": "critique", "verdict": {"verdict": "fail"}}}} self.assertFalse(gate.decide(self.status(), steps, "s5").proceed) def test_no_verdict(self): self.assertFalse(gate.decide(self.status(), {"steps": {"s1": {"phase": "plan"}}}, "s5").proceed) def test_a_failed_step(self): steps = {"steps": {"s2": {"phase": "collect", "status": "failed"}, "s4": {"phase": "c", "verdict": "pass"}}} self.assertFalse(gate.decide(self.status(), steps, "s5").proceed) def test_unparseable_verdict(self): steps = {"steps": {"s4": {"phase": "critique", "verdict": {"verdict": None}}}} self.assertFalse(gate.decide(self.status(), steps, "s5").proceed) class AuthTest(unittest.TestCase): def test_renews_before_expiry_and_hides_the_secret(self): clock = Clock() issued = [] def post(url, form): issued.append(form["grant_type"]) return {"access_token": f"tok{len(issued)}", "expires_in": 300} tokens = ClientCredentials("kab-automation", "s3cret", post=post, clock=clock) self.assertEqual(tokens.token(), "tok1") clock.now += 200 self.assertEqual(tokens.token(), "tok1") clock.now += 50 # inside the last minute self.assertEqual(tokens.token(), "tok2") self.assertEqual(issued, ["client_credentials", "client_credentials"]) self.assertNotIn("s3cret", repr(tokens)) def test_no_secret(self): with self.assertRaises(AuthError): ClientCredentials("kab-automation", "") def test_a_401_renews_the_token_once(self): calls = [] class T: def token(self, force=False): calls.append(force) return "x" replies = iter([(401, b""), (200, b'{"ok": 1}')]) kab = Kab(T(), transport=lambda *a: next(replies), sleep=lambda s: None) self.assertEqual(kab.call("GET", "/x"), {"ok": 1}) self.assertEqual(calls, [False, True]) if __name__ == "__main__": unittest.main()