- New cp1-discovery-package-agent (ADR 0020): one package per reviewed run, every part hashed (sha256 over canonical JSON), with a manifest and a value scorecard. - Applications grouped CMDB -> tag -> resource group with a judged model fallback (ADR 0021): CMDB 0.7.0 carries ownership, dependency mapping 0.6.0 groups, report 0.9.0 shows them. - Critic 0.6.1 accepts the producers' enums; build_workflow.py refuses a consumer enum narrower than its producer's. - Code hooks, sub-agents and tests for the inventory, CMDB, dependency, planner and critic agents; the report and Well-Architected review agents; ADRs 0012-0021. - Workflow cases with value bars and run limits; run evaluation and usage collection scripts; the run context template and the field-by-field review for ADR 0016. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
110 lines
5.1 KiB
Python
110 lines
5.1 KiB
Python
"""The workflow's upload step, split into the run context file and the files it lists (ADR 0018).
|
|
|
|
`parse_assessment_files` hands every step one string: the files in upload order, joined by FILE_SEPARATOR, with no
|
|
file names in the text. The run context file is always first (kab_run_workflow.py uploads it first), and its
|
|
`uploads:` list names the files after it, in order.
|
|
|
|
This file is identical in the planner, inventory, reconciliation and report agents (a test in each compares it with
|
|
the planner's). It knows no column of any file: a table is a header line and one line per row. No runtime imports.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
import yaml
|
|
|
|
FILE_SEPARATOR = "\n\n---\n\n" # data_file_processing.FILE_SEPARATOR
|
|
SHEET_HEADER = re.compile(r"^=== Sheet: (.*) ===$") # data_file_processing.SHEET_HEADER, one per XLSX sheet
|
|
EMPTY_FRAME = "Empty DataFrame" # pandas' rendering of a sheet or CSV with no rows
|
|
PARSE_ERRORS = ("Excel parsing error:", "CSV parsing error:", "PDF parsing error:", "Error parsing ",
|
|
"[Error parsing file:") # the platform returns a failed parse as text, not an error
|
|
ROW_INDEX = r"^\s*(\d+)\s" # pandas puts the row index first on each row line
|
|
|
|
|
|
class UploadUnreadable(ValueError):
|
|
"""A listed upload is missing, or the platform couldn't parse it."""
|
|
|
|
|
|
def _yaml(text: str) -> Dict[str, Any]:
|
|
try:
|
|
data = yaml.safe_load(text)
|
|
except yaml.YAMLError:
|
|
return {}
|
|
return data if isinstance(data, dict) else {}
|
|
|
|
|
|
def _entry(item: Any) -> Dict[str, Any]:
|
|
if isinstance(item, dict):
|
|
return {"name": str(item.get("name") or ""), "source": str(item.get("source") or "").lower() or None,
|
|
"format": item.get("format")}
|
|
return {"name": str(item), "source": None, "format": None}
|
|
|
|
|
|
def split(parsed_content: Optional[str]) -> Dict[str, Any]:
|
|
"""{"context": the run context as a dict ({} if it isn't YAML), "context_text": its text,
|
|
"uploads": [{"name", "source", "format", "text"}] for each entry in its `uploads:`, in order}.
|
|
|
|
The text is split at most once per listed upload, so a separator inside the last upload never cuts it short.
|
|
A listed upload with no part of its own has text None.
|
|
"""
|
|
text = parsed_content or ""
|
|
first = text.split(FILE_SEPARATOR, 1)[0]
|
|
doc = _yaml(first)
|
|
listed = doc.get("uploads") if isinstance(doc.get("uploads"), list) else []
|
|
parts = text.split(FILE_SEPARATOR, len(listed)) if listed else [text]
|
|
uploads = []
|
|
for index, item in enumerate(listed, start=1):
|
|
entry = _entry(item)
|
|
entry["text"] = parts[index] if index < len(parts) else None
|
|
uploads.append(entry)
|
|
return {"context": doc, "context_text": parts[0], "uploads": uploads}
|
|
|
|
|
|
def context_section(parsed_content: Optional[str]) -> Dict[str, Any]:
|
|
"""The run context's `context:` section, or {}."""
|
|
section = split(parsed_content)["context"].get("context")
|
|
return section if isinstance(section, dict) else {}
|
|
|
|
|
|
def uploads_from(parsed_content: Optional[str], source: str) -> List[Dict[str, Any]]:
|
|
"""The listed uploads whose `source:` is this one (e.g. cmdb)."""
|
|
return [u for u in split(parsed_content)["uploads"] if u["source"] == source]
|
|
|
|
|
|
def _rows(lines: List[str]) -> List[Tuple[int, str]]:
|
|
found = [re.match(ROW_INDEX, line) for line in lines]
|
|
numbers = [int(m.group(1)) for m in found if m]
|
|
if len(numbers) == len(lines) and len(set(numbers)) == len(numbers):
|
|
return list(zip(numbers, lines))
|
|
return list(enumerate(lines)) # not a pandas table (a raw CSV, say): number rows by position
|
|
|
|
|
|
def _table(sheet: Optional[str], lines: List[str]) -> Dict[str, Any]:
|
|
lines = [line.rstrip() for line in lines if line.strip()]
|
|
if not lines or lines[0].strip().startswith(EMPTY_FRAME):
|
|
return {"sheet": sheet, "header": lines[0] if lines else "", "rows": []}
|
|
return {"sheet": sheet, "header": lines[0], "rows": _rows(lines[1:])}
|
|
|
|
|
|
def tables(upload: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
"""[{"sheet": name or None, "header": the header line, "rows": [(row number, line)]}] for one upload.
|
|
Raises UploadUnreadable for a missing upload or one the platform failed to parse."""
|
|
text = upload.get("text")
|
|
if text is None or not text.strip():
|
|
raise UploadUnreadable(f"{upload.get('name')}: listed in the run context's uploads but not uploaded")
|
|
if text.strip().startswith(PARSE_ERRORS):
|
|
raise UploadUnreadable(f"{upload.get('name')}: the platform could not parse it: {text.strip()[:200]}")
|
|
sheets: List[Tuple[Optional[str], List[str]]] = []
|
|
current: Tuple[Optional[str], List[str]] = (None, [])
|
|
for line in text.split("\n"):
|
|
header = SHEET_HEADER.match(line.strip())
|
|
if header:
|
|
if current[0] is not None or any(x.strip() for x in current[1]):
|
|
sheets.append(current)
|
|
current = (header.group(1), [])
|
|
else:
|
|
current[1].append(line)
|
|
sheets.append(current)
|
|
return [_table(sheet, lines) for sheet, lines in sheets]
|