张宗平
4c4a8f6435
- src/tsmm/eval/parse_answer.py: text→JSON segments→point scores (JSON-first, regex fallback, all-zero on fail). - src/tsmm/eval/ts_metrics.py: VUS-PR (main, threshold-free; constant-score → prevalence, NOT inflated), AUC-PR, Point-F1 (no PA), PA-F1 (control only), self-contained Affiliation-F1. - src/tsmm/eval/qa_judge.py: RuleJudge (anomaly IoU / describe+forecast numeric tolerance) + LLMJudge (stub backend = deterministic offline heuristic for the no-network host; openai backend = GPT-4o 0-5; pluggable callable). - src/tsmm/eval/baselines.py: trivial (Random/Constant/AllReport) + ts_to_text + pure_llm_answer glue; Time-LLM/ChatTS marked not-reproduced (honest). - src/tsmm/eval/report.py + scripts/run_eval.sh: trained model + all baselines through the SAME pipeline → reports/eval-YYYYMMDD.md (VUS-PR main, PA-F1 explicitly labelled control, trivial baselines included). - tests: 45 new (parse 13, metrics 12, judge 12, baselines 8). 145 passing. - Smoke: run_eval on 40 samples produces valid markdown; PA-F1≈1.0 vs VUS-PR≈0.35 for all_report demonstrates why PA must not be the headline.
106 lines
3.2 KiB
Python
106 lines
3.2 KiB
Python
"""Tests for eval/qa_judge.py (T5.3)."""
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from tsmm.eval.qa_judge import (
|
|
RuleJudge,
|
|
rule_score_describe,
|
|
rule_score_anomaly,
|
|
rule_score_forecast,
|
|
)
|
|
|
|
|
|
class TestDescribeRule:
|
|
def test_numerical_within_tolerance(self):
|
|
# predicted mean within 5% of reference → 1.0
|
|
out = rule_score_describe(
|
|
pred='{"mean": 50.0, "std": 2.0}',
|
|
ref={"mean": 50.5, "std": 2.1},
|
|
rtol=0.05,
|
|
)
|
|
assert out["score"] == 1.0
|
|
assert out["parsed"] is True
|
|
|
|
def test_outside_tolerance(self):
|
|
out = rule_score_describe(
|
|
pred='{"mean": 80.0}',
|
|
ref={"mean": 50.0},
|
|
rtol=0.05,
|
|
)
|
|
assert out["score"] == 0.0
|
|
assert out["parsed"] is True
|
|
|
|
def test_unparseable(self):
|
|
out = rule_score_describe(pred="the mean is around 50", ref={"mean": 50.0})
|
|
assert out["parsed"] is False
|
|
assert out["score"] == 0.0
|
|
|
|
|
|
class TestAnomalyRule:
|
|
def test_exact_match(self):
|
|
pred = '{"segments": [[10, 20]]}'
|
|
ref_segs = [(10, 20)]
|
|
out = rule_score_anomaly(pred, ref_segs, T=50)
|
|
assert out["score"] == 1.0
|
|
assert out["parsed"] is True
|
|
|
|
def test_iou_partial(self):
|
|
pred = '{"segments": [[10, 30]]}'
|
|
ref_segs = [(20, 40)]
|
|
out = rule_score_anomaly(pred, ref_segs, T=50)
|
|
assert 0.0 < out["score"] < 1.0
|
|
|
|
def test_no_overlap(self):
|
|
pred = '{"segments": [[0, 5]]}'
|
|
ref_segs = [(20, 40)]
|
|
out = rule_score_anomaly(pred, ref_segs, T=50)
|
|
assert out["score"] == 0.0
|
|
|
|
|
|
class TestForecastRule:
|
|
def test_within_tolerance(self):
|
|
pred = '{"forecast": [1.0, 2.0, 3.0]}'
|
|
ref = [1.05, 1.95, 3.1]
|
|
out = rule_score_forecast(pred, ref, rtol=0.1)
|
|
assert out["score"] == 1.0
|
|
assert out["parsed"] is True
|
|
|
|
def test_one_off(self):
|
|
pred = '{"forecast": [1.0, 2.0, 9.0]}'
|
|
ref = [1.0, 2.0, 3.0]
|
|
out = rule_score_forecast(pred, ref, rtol=0.1)
|
|
assert out["score"] < 1.0
|
|
|
|
|
|
class TestRuleJudgeDispatch:
|
|
def test_judge_anomaly(self):
|
|
judge = RuleJudge()
|
|
out = judge.judge("anomaly", '{"segments": [[5, 8]]}', ref_segs=[(5, 8)], T=20)
|
|
assert out["score"] == 1.0
|
|
assert out["track"] == "rule"
|
|
|
|
def test_judge_unknown_category_skips(self):
|
|
judge = RuleJudge()
|
|
out = judge.judge("root_cause", "some text answer", ref_answer="some text answer")
|
|
assert out["track"] == "skip"
|
|
assert out["score"] is None
|
|
|
|
|
|
class TestLLMJudgeStub:
|
|
def test_stub_returns_deterministic(self):
|
|
from tsmm.eval.qa_judge import LLMJudge
|
|
judge = LLMJudge(backend="stub")
|
|
s1 = judge.score("Q?", "ref", "pred")
|
|
s2 = judge.score("Q?", "ref", "pred")
|
|
assert isinstance(s1, dict)
|
|
assert "score" in s1 and 0 <= s1["score"] <= 5
|
|
assert s1 == s2 # deterministic stub
|
|
|
|
def test_stub_rewards_exact_match(self):
|
|
from tsmm.eval.qa_judge import LLMJudge
|
|
judge = LLMJudge(backend="stub")
|
|
same = judge.score("Q?", "正常", "正常")["score"]
|
|
diff = judge.score("Q?", "正常", "完全不同的乱码回答xyz")["score"]
|
|
assert same >= diff
|