58 lines
2.0 KiB
Python
58 lines
2.0 KiB
Python
"""ADK Evaluation module for GCP Solution Architecture Agent.
|
|
|
|
Uses google.adk.evaluation.Evaluator with PostgreSQL database persistence.
|
|
"""
|
|
|
|
import logging
|
|
import uuid
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
from app.adk.compat import Evaluator
|
|
from app.adk.runners import ADKAgentRunner
|
|
from app.database import get_db_manager
|
|
from eval.metrics import evaluate_case_run
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ADKEvaluator(Evaluator):
|
|
"""ADK Evaluator persisting evaluation results to PostgreSQL."""
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
self.runner = ADKAgentRunner()
|
|
self.db_manager = get_db_manager()
|
|
|
|
def evaluate_benchmark_case(self, case: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Execute and score benchmark case via ADK runner and record to PostgreSQL."""
|
|
eval_id = str(uuid.uuid4())
|
|
case_id = case.get("id", "case-unknown")
|
|
req_summary = case.get("workflow_request", "")
|
|
|
|
run_res = self.runner.run_execution(request_summary=req_summary)
|
|
artifacts = run_res.get("artifacts", {})
|
|
|
|
state_for_metrics = {
|
|
"requirements_doc": artifacts.get("requirements_doc", ""),
|
|
"architecture_doc": artifacts.get("architecture_doc", ""),
|
|
"mermaid_diagram": artifacts.get("mermaid_diagram", ""),
|
|
"terraform_code": artifacts.get("terraform_code", ""),
|
|
"solution_guide": artifacts.get("solution_guide", ""),
|
|
}
|
|
|
|
eval_result = evaluate_case_run(state_for_metrics, case)
|
|
|
|
# Save evaluation to PostgreSQL database
|
|
self.db_manager.save_evaluation(
|
|
eval_id=eval_id,
|
|
case_id=case_id,
|
|
total_score=eval_result["total_score"],
|
|
max_score=eval_result["max_score"],
|
|
pass_rate=eval_result["percentage"],
|
|
passed=eval_result["passed"],
|
|
details=eval_result,
|
|
)
|
|
|
|
logger.info("Recorded ADK evaluation %s for case %s (Score: %.1f%%)", eval_id, case_id, eval_result["percentage"])
|
|
return eval_result
|