feat(m5): full eval pipeline — parse/metrics/judge/baselines/report (T5.1-T5.5)
- src/tsmm/eval/parse_answer.py: text→JSON segments→point scores (JSON-first, regex fallback, all-zero on fail). - src/tsmm/eval/ts_metrics.py: VUS-PR (main, threshold-free; constant-score → prevalence, NOT inflated), AUC-PR, Point-F1 (no PA), PA-F1 (control only), self-contained Affiliation-F1. - src/tsmm/eval/qa_judge.py: RuleJudge (anomaly IoU / describe+forecast numeric tolerance) + LLMJudge (stub backend = deterministic offline heuristic for the no-network host; openai backend = GPT-4o 0-5; pluggable callable). - src/tsmm/eval/baselines.py: trivial (Random/Constant/AllReport) + ts_to_text + pure_llm_answer glue; Time-LLM/ChatTS marked not-reproduced (honest). - src/tsmm/eval/report.py + scripts/run_eval.sh: trained model + all baselines through the SAME pipeline → reports/eval-YYYYMMDD.md (VUS-PR main, PA-F1 explicitly labelled control, trivial baselines included). - tests: 45 new (parse 13, metrics 12, judge 12, baselines 8). 145 passing. - Smoke: run_eval on 40 samples produces valid markdown; PA-F1≈1.0 vs VUS-PR≈0.35 for all_report demonstrates why PA must not be the headline.
This commit is contained in:
@@ -0,0 +1,105 @@
|
||||
"""Tests for eval/qa_judge.py (T5.3)."""
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tsmm.eval.qa_judge import (
|
||||
RuleJudge,
|
||||
rule_score_describe,
|
||||
rule_score_anomaly,
|
||||
rule_score_forecast,
|
||||
)
|
||||
|
||||
|
||||
class TestDescribeRule:
|
||||
def test_numerical_within_tolerance(self):
|
||||
# predicted mean within 5% of reference → 1.0
|
||||
out = rule_score_describe(
|
||||
pred='{"mean": 50.0, "std": 2.0}',
|
||||
ref={"mean": 50.5, "std": 2.1},
|
||||
rtol=0.05,
|
||||
)
|
||||
assert out["score"] == 1.0
|
||||
assert out["parsed"] is True
|
||||
|
||||
def test_outside_tolerance(self):
|
||||
out = rule_score_describe(
|
||||
pred='{"mean": 80.0}',
|
||||
ref={"mean": 50.0},
|
||||
rtol=0.05,
|
||||
)
|
||||
assert out["score"] == 0.0
|
||||
assert out["parsed"] is True
|
||||
|
||||
def test_unparseable(self):
|
||||
out = rule_score_describe(pred="the mean is around 50", ref={"mean": 50.0})
|
||||
assert out["parsed"] is False
|
||||
assert out["score"] == 0.0
|
||||
|
||||
|
||||
class TestAnomalyRule:
|
||||
def test_exact_match(self):
|
||||
pred = '{"segments": [[10, 20]]}'
|
||||
ref_segs = [(10, 20)]
|
||||
out = rule_score_anomaly(pred, ref_segs, T=50)
|
||||
assert out["score"] == 1.0
|
||||
assert out["parsed"] is True
|
||||
|
||||
def test_iou_partial(self):
|
||||
pred = '{"segments": [[10, 30]]}'
|
||||
ref_segs = [(20, 40)]
|
||||
out = rule_score_anomaly(pred, ref_segs, T=50)
|
||||
assert 0.0 < out["score"] < 1.0
|
||||
|
||||
def test_no_overlap(self):
|
||||
pred = '{"segments": [[0, 5]]}'
|
||||
ref_segs = [(20, 40)]
|
||||
out = rule_score_anomaly(pred, ref_segs, T=50)
|
||||
assert out["score"] == 0.0
|
||||
|
||||
|
||||
class TestForecastRule:
|
||||
def test_within_tolerance(self):
|
||||
pred = '{"forecast": [1.0, 2.0, 3.0]}'
|
||||
ref = [1.05, 1.95, 3.1]
|
||||
out = rule_score_forecast(pred, ref, rtol=0.1)
|
||||
assert out["score"] == 1.0
|
||||
assert out["parsed"] is True
|
||||
|
||||
def test_one_off(self):
|
||||
pred = '{"forecast": [1.0, 2.0, 9.0]}'
|
||||
ref = [1.0, 2.0, 3.0]
|
||||
out = rule_score_forecast(pred, ref, rtol=0.1)
|
||||
assert out["score"] < 1.0
|
||||
|
||||
|
||||
class TestRuleJudgeDispatch:
|
||||
def test_judge_anomaly(self):
|
||||
judge = RuleJudge()
|
||||
out = judge.judge("anomaly", '{"segments": [[5, 8]]}', ref_segs=[(5, 8)], T=20)
|
||||
assert out["score"] == 1.0
|
||||
assert out["track"] == "rule"
|
||||
|
||||
def test_judge_unknown_category_skips(self):
|
||||
judge = RuleJudge()
|
||||
out = judge.judge("root_cause", "some text answer", ref_answer="some text answer")
|
||||
assert out["track"] == "skip"
|
||||
assert out["score"] is None
|
||||
|
||||
|
||||
class TestLLMJudgeStub:
|
||||
def test_stub_returns_deterministic(self):
|
||||
from tsmm.eval.qa_judge import LLMJudge
|
||||
judge = LLMJudge(backend="stub")
|
||||
s1 = judge.score("Q?", "ref", "pred")
|
||||
s2 = judge.score("Q?", "ref", "pred")
|
||||
assert isinstance(s1, dict)
|
||||
assert "score" in s1 and 0 <= s1["score"] <= 5
|
||||
assert s1 == s2 # deterministic stub
|
||||
|
||||
def test_stub_rewards_exact_match(self):
|
||||
from tsmm.eval.qa_judge import LLMJudge
|
||||
judge = LLMJudge(backend="stub")
|
||||
same = judge.score("Q?", "正常", "正常")["score"]
|
||||
diff = judge.score("Q?", "正常", "完全不同的乱码回答xyz")["score"]
|
||||
assert same >= diff
|
||||
Reference in New Issue
Block a user