feat: working orhcestration engine
This commit is contained in:
125
scripts/bring.py
Executable file
125
scripts/bring.py
Executable file
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bring modules across from the reference, and record where each came from.
|
||||
|
||||
scripts/bring.py <reference checkout> <source path> [<source path> ...]
|
||||
|
||||
A source path is relative to the reference, under `app/` or `tests/`. Each file is
|
||||
copied to the matching place here (`app/x.py` becomes `execution_engine/x.py`) with
|
||||
its imports of `app` renamed, and nothing else changed. A module that is about the
|
||||
reference's own subject is not part of the engine: it goes to
|
||||
`tests/reference_workflow/`, the example Workflow the engine's tests run (ADR-0001). REFERENCE.lock.json records,
|
||||
for every file brought, the reference's commit and the hash of the file as it stood
|
||||
there, so a later look at the reference can tell which of them have moved since.
|
||||
|
||||
The reference is only read. It must have no uncommitted change to a file brought,
|
||||
or the commit recorded would not describe what was copied.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent.parent
|
||||
LOCK = HERE / "REFERENCE.lock.json"
|
||||
PACKAGE = "execution_engine"
|
||||
# The reference's modules that are about its subject, and where each goes.
|
||||
EXAMPLE = "tests/reference_workflow"
|
||||
EXAMPLE_MODULES = {
|
||||
"app/workflows/chain.py": "chain.py",
|
||||
"app/workflows/target_architecture.py": "target_architecture.py",
|
||||
"app/workflows/changes.py": "changes.py",
|
||||
"app/workflows/work_breakdown.py": "work_breakdown.py",
|
||||
"app/tools/pillars.py": "pillars.py",
|
||||
"app/tools/meaning.py": "meaning.py",
|
||||
"app/tools/local.py": "local.py",
|
||||
"app/tools/modules.py": "modules.py",
|
||||
"app/tools/coverage.py": "coverage.py",
|
||||
"app/tools/design_review.py": "design_review.py",
|
||||
"app/tools/requirements_review.py": "requirements_review.py",
|
||||
"app/tools/schema_review.py": "schema_review.py",
|
||||
"app/tools/assessment.py": "assessment.py",
|
||||
"app/presentation.py": "presentation.py",
|
||||
}
|
||||
_IMPORT = re.compile(r"^(\s*)(from|import) app(\.|\s)", re.MULTILINE)
|
||||
_DOTTED = re.compile(r"(?<![\w.])app\.(?=(?:[a-z_]+\.)*[a-z_]+\b)")
|
||||
|
||||
|
||||
def git(reference: Path, *args: str) -> str:
|
||||
return subprocess.run(
|
||||
["git", "-C", str(reference), *args],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
|
||||
|
||||
def _dotted(path: str) -> str:
|
||||
return path.removesuffix(".py").replace("/", ".")
|
||||
|
||||
|
||||
def destination(source: str) -> Path:
|
||||
if source in EXAMPLE_MODULES:
|
||||
return HERE / EXAMPLE / EXAMPLE_MODULES[source]
|
||||
if source.startswith("app/"):
|
||||
return HERE / PACKAGE / source.removeprefix("app/")
|
||||
if source.startswith("tests/"):
|
||||
return HERE / source
|
||||
raise SystemExit(f"{source}: only files under app/ or tests/ are brought")
|
||||
|
||||
|
||||
def renamed(text: str, is_test: bool = False) -> str:
|
||||
# The example's modules first, longest name first, so the general rule below
|
||||
# does not send them into the engine.
|
||||
example = {
|
||||
_dotted(source): _dotted(f"{EXAMPLE}/{name}")
|
||||
for source, name in EXAMPLE_MODULES.items()
|
||||
}
|
||||
if is_test:
|
||||
# A test that named the reference's Deliverables now takes them from the
|
||||
# example Workflow, where the reference's graph is declared.
|
||||
example["app.workflows.deliverables"] = _dotted(f"{EXAMPLE}/workflow")
|
||||
for source in sorted(example, key=len, reverse=True):
|
||||
text = re.sub(rf"(?<![\w.]){re.escape(source)}\b", example[source], text)
|
||||
text = _IMPORT.sub(rf"\1\2 {PACKAGE}\3", text)
|
||||
# Dotted names in strings and comments: "app.scope", `python -m app.scope`.
|
||||
return _DOTTED.sub(f"{PACKAGE}.", text)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) < 2:
|
||||
print(__doc__)
|
||||
return 2
|
||||
reference = Path(argv[0]).resolve()
|
||||
sources = argv[1:]
|
||||
dirty = git(reference, "status", "--porcelain", "--", *sources)
|
||||
if dirty:
|
||||
raise SystemExit(f"the reference has uncommitted changes to:\n{dirty}")
|
||||
commit = git(reference, "rev-parse", "HEAD")
|
||||
|
||||
lock = json.loads(LOCK.read_text(encoding="utf-8")) if LOCK.is_file() else {}
|
||||
lock.setdefault("reference", git(reference, "remote", "get-url", "origin"))
|
||||
files = lock.setdefault("files", {})
|
||||
for source in sources:
|
||||
original = (reference / source).read_bytes()
|
||||
target = destination(source)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
text = renamed(original.decode("utf-8"), source.startswith("tests/"))
|
||||
target.write_text(text, encoding="utf-8")
|
||||
files[str(target.relative_to(HERE))] = {
|
||||
"source": source,
|
||||
"commit": commit,
|
||||
"sha256": hashlib.sha256(original).hexdigest(),
|
||||
}
|
||||
print(f"{source} -> {target.relative_to(HERE)}")
|
||||
lock["files"] = dict(sorted(files.items()))
|
||||
LOCK.write_text(json.dumps(lock, indent=2) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
110
scripts/repoint.py
Normal file
110
scripts/repoint.py
Normal file
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Point a test brought from the reference at the example Workflow.
|
||||
|
||||
scripts/repoint.py tests/execution/test_x.py [...]
|
||||
|
||||
The reference's tests call its engine with no Workflow, because it has only one.
|
||||
Here every such call names the example: `Ledger(...)`, `Ledger.from_dict(...)` and
|
||||
`specialist_tools(...)` gain `WORKFLOW` as their first argument, the engine is built
|
||||
with `workflow=WORKFLOW`, and the sample data and sample submissions are taken from
|
||||
the example. Running it twice changes nothing more.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
EXAMPLE = "tests.reference_workflow.workflow"
|
||||
_CALLS = (
|
||||
(re.compile(r"\bLedger\.from_dict\((?!WORKFLOW)"), "Ledger.from_dict(WORKFLOW, "),
|
||||
(re.compile(r"(?<![\w.])Ledger\((?!WORKFLOW)"), "Ledger(WORKFLOW, "),
|
||||
(re.compile(r"\bspecialist_tools\((?!\s*WORKFLOW)"), "specialist_tools(WORKFLOW, "),
|
||||
(
|
||||
re.compile(
|
||||
r"\b(ExecutionEngine|build_execution_graph|make_execution_nodes)\((?!\s*workflow=)"
|
||||
),
|
||||
r"\1(workflow=WORKFLOW, ",
|
||||
),
|
||||
)
|
||||
# Names the engine no longer has, and where the example keeps them.
|
||||
_MOVED = (
|
||||
(
|
||||
re.compile(
|
||||
r"^from execution_engine\.nodes\.execution_nodes import GUIDE_FILENAME\n",
|
||||
re.MULTILINE,
|
||||
),
|
||||
"from tests.reference_workflow.hooks import GUIDE_FILENAME\n",
|
||||
),
|
||||
(re.compile(r"\bSolutionArchitectExecutor\b"), "EngineExecutor"),
|
||||
)
|
||||
_FROM_EXAMPLE = re.compile(
|
||||
rf"^\s*from {re.escape(EXAMPLE)} import (\([^)]*\)|[^\n]+)", re.MULTILINE
|
||||
)
|
||||
_TOP_IMPORT = re.compile(r"^(?:from|import) .*$", re.MULTILINE)
|
||||
# What the reference's tests take from its scripted runtime that is the example's
|
||||
# here: its sample data, and a sample submission of one of its Deliverables.
|
||||
_EXAMPLES_OWN = ("SAMPLE_DATA", "sample_submission")
|
||||
_FAKE_IMPORT = re.compile(
|
||||
r"^from execution_engine\.adk\.fake_runtime import (\([^)]*\)|[^\n]+)\n",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def _move_examples_own(text: str) -> str:
|
||||
"""Take the example's own names out of an import from the scripted runtime, and
|
||||
import them from the example."""
|
||||
match = _FAKE_IMPORT.search(text)
|
||||
if not match:
|
||||
return text
|
||||
items = [
|
||||
item.strip() for item in match.group(1).strip("()").split(",") if item.strip()
|
||||
]
|
||||
moved = [item for item in items if item.split(" as ")[0] in _EXAMPLES_OWN]
|
||||
if not moved:
|
||||
return text
|
||||
kept = [item for item in items if item not in moved]
|
||||
lines = (
|
||||
f"from execution_engine.adk.fake_runtime import {', '.join(kept)}\n"
|
||||
if kept
|
||||
else ""
|
||||
)
|
||||
lines += f"from {EXAMPLE} import {', '.join(moved)}\n"
|
||||
return text[: match.start()] + lines + text[match.end() :]
|
||||
|
||||
|
||||
def _end_of_statement(text: str, start: int) -> int:
|
||||
"""Where the import statement starting at `start` ends."""
|
||||
line_end = text.index("\n", start)
|
||||
if "(" in text[start:line_end] and ")" not in text[start:line_end]:
|
||||
return text.index("\n", text.index(")", start))
|
||||
return line_end
|
||||
|
||||
|
||||
def repoint(text: str) -> str:
|
||||
for pattern, replacement in _CALLS:
|
||||
text = pattern.sub(replacement, text)
|
||||
text = text.replace("(WORKFLOW, )", "(WORKFLOW)")
|
||||
for pattern, replacement in _MOVED:
|
||||
text = pattern.sub(replacement, text)
|
||||
text = _move_examples_own(text)
|
||||
|
||||
imported = " ".join(_FROM_EXAMPLE.findall(text))
|
||||
missing = [
|
||||
name
|
||||
for name in ("WORKFLOW",)
|
||||
if re.search(rf"\b{name}\b", text) and not re.search(rf"\b{name}\b", imported)
|
||||
]
|
||||
if missing:
|
||||
line = f"from {EXAMPLE} import {', '.join(missing)}\n"
|
||||
imports = list(_TOP_IMPORT.finditer(text))
|
||||
at = _end_of_statement(text, imports[-1].start()) + 1 if imports else 0
|
||||
text = text[:at] + line + text[at:]
|
||||
return text
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for name in sys.argv[1:]:
|
||||
path = Path(name)
|
||||
path.write_text(repoint(path.read_text(encoding="utf-8")), encoding="utf-8")
|
||||
Reference in New Issue
Block a user