feat(m5): full eval pipeline — parse/metrics/judge/baselines/report (T5.1-T5.5)
- src/tsmm/eval/parse_answer.py: text→JSON segments→point scores (JSON-first, regex fallback, all-zero on fail). - src/tsmm/eval/ts_metrics.py: VUS-PR (main, threshold-free; constant-score → prevalence, NOT inflated), AUC-PR, Point-F1 (no PA), PA-F1 (control only), self-contained Affiliation-F1. - src/tsmm/eval/qa_judge.py: RuleJudge (anomaly IoU / describe+forecast numeric tolerance) + LLMJudge (stub backend = deterministic offline heuristic for the no-network host; openai backend = GPT-4o 0-5; pluggable callable). - src/tsmm/eval/baselines.py: trivial (Random/Constant/AllReport) + ts_to_text + pure_llm_answer glue; Time-LLM/ChatTS marked not-reproduced (honest). - src/tsmm/eval/report.py + scripts/run_eval.sh: trained model + all baselines through the SAME pipeline → reports/eval-YYYYMMDD.md (VUS-PR main, PA-F1 explicitly labelled control, trivial baselines included). - tests: 45 new (parse 13, metrics 12, judge 12, baselines 8). 145 passing. - Smoke: run_eval on 40 samples produces valid markdown; PA-F1≈1.0 vs VUS-PR≈0.35 for all_report demonstrates why PA must not be the headline.
This commit is contained in:
@@ -0,0 +1,109 @@
|
||||
"""Tests for eval/ts_metrics.py (T5.2)."""
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tsmm.eval.ts_metrics import (
|
||||
affiliation_f1,
|
||||
auc_pr,
|
||||
point_f1,
|
||||
point_f1_with_pa,
|
||||
vus_pr,
|
||||
)
|
||||
|
||||
|
||||
def labels(segs, T=100):
|
||||
y = np.zeros(T, dtype=int)
|
||||
for a, b in segs:
|
||||
y[a:b + 1] = 1
|
||||
return y
|
||||
|
||||
|
||||
class TestPointF1:
|
||||
def test_perfect(self):
|
||||
y = labels([(20, 40)], T=100)
|
||||
s = np.zeros(100); s[20:41] = 1.0
|
||||
f1, prec, rec = point_f1(s, y)
|
||||
assert f1 == pytest.approx(1.0, abs=1e-6)
|
||||
|
||||
def test_no_overlap(self):
|
||||
y = labels([(20, 40)])
|
||||
s = np.zeros(100); s[60:70] = 1.0
|
||||
f1, _, _ = point_f1(s, y)
|
||||
assert f1 == pytest.approx(0.0, abs=1e-6)
|
||||
|
||||
def test_partial(self):
|
||||
y = labels([(20, 40)])
|
||||
s = np.zeros(100); s[30:41] = 0.9
|
||||
f1, prec, rec = point_f1(s, y, threshold=0.5)
|
||||
# predicted 11 anomalous all in true region: precision 1, recall 11/21
|
||||
assert prec == pytest.approx(1.0, abs=1e-6)
|
||||
assert rec == pytest.approx(11 / 21, abs=1e-3)
|
||||
|
||||
|
||||
class TestAUCPR:
|
||||
def test_perfect_ranking(self):
|
||||
y = labels([(10, 20)])
|
||||
s = np.zeros(100); s[10:21] = 1.0
|
||||
assert auc_pr(s, y) == pytest.approx(1.0, abs=1e-6)
|
||||
|
||||
def test_inverse_ranking_low(self):
|
||||
y = labels([(10, 20)])
|
||||
s = np.ones(100); s[10:21] = 0.0 # anomalies score LOW
|
||||
assert auc_pr(s, y) < 0.5
|
||||
|
||||
|
||||
class TestPAF1:
|
||||
def test_pa_inflates(self):
|
||||
# PA: if ANY predicted point is in the true anomaly, the whole segment
|
||||
# counts as caught → much higher F1 than point-wise
|
||||
y = labels([(20, 40)])
|
||||
s = np.zeros(100); s[40] = 1.0 # single overlap point
|
||||
f1_pa, _, _ = point_f1_with_pa(s, y)
|
||||
f1_pt, _, _ = point_f1(s, y)
|
||||
assert f1_pa > f1_pt
|
||||
|
||||
|
||||
class TestAffiliationF1:
|
||||
def test_perfect(self):
|
||||
y = labels([(20, 40)])
|
||||
s = np.zeros(100); s[20:41] = 1.0
|
||||
f1, _, _ = affiliation_f1(s, y)
|
||||
assert f1 == pytest.approx(1.0, abs=1e-6)
|
||||
|
||||
def test_far_predictions_lower(self):
|
||||
y = labels([(20, 40)])
|
||||
s_near = np.zeros(100); s_near[41:45] = 1.0
|
||||
s_far = np.zeros(100); s_far[90:95] = 1.0
|
||||
f1_near, _, _ = affiliation_f1(s_near, y)
|
||||
f1_far, _, _ = affiliation_f1(s_far, y)
|
||||
assert f1_near > f1_far
|
||||
|
||||
|
||||
class TestVUSPR:
|
||||
def test_perfect_high(self):
|
||||
y = labels([(20, 40)])
|
||||
s = np.zeros(100); s[20:41] = 1.0
|
||||
v = vus_pr(s, y)
|
||||
assert v == pytest.approx(1.0, abs=1e-3)
|
||||
|
||||
def test_random_scores_moderate(self):
|
||||
rng = np.random.default_rng(0)
|
||||
y = labels([(20, 40)])
|
||||
s = rng.random(100)
|
||||
v = vus_pr(s, y)
|
||||
# random ranking → VUS-PR around the anomaly prevalence (~0.21)
|
||||
assert 0.0 < v < 0.6
|
||||
|
||||
def test_all_report_not_inflated(self):
|
||||
# spec REQUIREMENT: "全报异常" trivial baseline must NOT have inflated VUS-PR
|
||||
y = labels([(20, 40)], T=100)
|
||||
s = np.ones(100) # report everything as anomalous
|
||||
v = vus_pr(s, y)
|
||||
# prevalence = 21/100; all-ones gives AUC-PR == prevalence everywhere
|
||||
assert v == pytest.approx(21 / 100, abs=1e-3)
|
||||
|
||||
def test_returns_scalar(self):
|
||||
y = labels([(5, 10)])
|
||||
s = np.random.default_rng(1).random(100)
|
||||
v = vus_pr(s, y)
|
||||
assert np.isscalar(v) or v.shape == ()
|
||||
Reference in New Issue
Block a user