33 lines
1021 B
Python
33 lines
1021 B
Python
"""Tests for Evaluation Harness and Metrics."""
|
|
|
|
import pytest
|
|
from eval.eval_harness import EvalHarness
|
|
from eval.optimizer import SkillOptimizer
|
|
|
|
|
|
def test_eval_harness_benchmark():
|
|
"""Run EvalHarness against benchmark cases."""
|
|
harness = EvalHarness()
|
|
summary = harness.run_eval_suite()
|
|
|
|
assert summary["total_cases"] >= 1
|
|
assert summary["passed_cases"] >= 1
|
|
assert summary["pass_rate_percentage"] == 100.0
|
|
|
|
|
|
def test_skill_optimizer_recommendations():
|
|
"""Verify optimizer generates tuning recommendations for simulated failures."""
|
|
optimizer = SkillOptimizer()
|
|
simulated_summary = {
|
|
"results": [
|
|
{
|
|
"case_id": "test-case",
|
|
"passed": False,
|
|
"feedback": ["Product selection was not explicitly deferred in discovery phase."],
|
|
}
|
|
]
|
|
}
|
|
recs = optimizer.generate_tuning_recommendations(simulated_summary)
|
|
assert len(recs) == 1
|
|
assert recs[0]["target_skill"] == "requirements_discovery"
|