张宗平
4c4a8f6435
- src/tsmm/eval/parse_answer.py: text→JSON segments→point scores (JSON-first, regex fallback, all-zero on fail). - src/tsmm/eval/ts_metrics.py: VUS-PR (main, threshold-free; constant-score → prevalence, NOT inflated), AUC-PR, Point-F1 (no PA), PA-F1 (control only), self-contained Affiliation-F1. - src/tsmm/eval/qa_judge.py: RuleJudge (anomaly IoU / describe+forecast numeric tolerance) + LLMJudge (stub backend = deterministic offline heuristic for the no-network host; openai backend = GPT-4o 0-5; pluggable callable). - src/tsmm/eval/baselines.py: trivial (Random/Constant/AllReport) + ts_to_text + pure_llm_answer glue; Time-LLM/ChatTS marked not-reproduced (honest). - src/tsmm/eval/report.py + scripts/run_eval.sh: trained model + all baselines through the SAME pipeline → reports/eval-YYYYMMDD.md (VUS-PR main, PA-F1 explicitly labelled control, trivial baselines included). - tests: 45 new (parse 13, metrics 12, judge 12, baselines 8). 145 passing. - Smoke: run_eval on 40 samples produces valid markdown; PA-F1≈1.0 vs VUS-PR≈0.35 for all_report demonstrates why PA must not be the headline.
110 lines
3.3 KiB
Python
110 lines
3.3 KiB
Python
"""Tests for eval/ts_metrics.py (T5.2)."""
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from tsmm.eval.ts_metrics import (
|
|
affiliation_f1,
|
|
auc_pr,
|
|
point_f1,
|
|
point_f1_with_pa,
|
|
vus_pr,
|
|
)
|
|
|
|
|
|
def labels(segs, T=100):
|
|
y = np.zeros(T, dtype=int)
|
|
for a, b in segs:
|
|
y[a:b + 1] = 1
|
|
return y
|
|
|
|
|
|
class TestPointF1:
|
|
def test_perfect(self):
|
|
y = labels([(20, 40)], T=100)
|
|
s = np.zeros(100); s[20:41] = 1.0
|
|
f1, prec, rec = point_f1(s, y)
|
|
assert f1 == pytest.approx(1.0, abs=1e-6)
|
|
|
|
def test_no_overlap(self):
|
|
y = labels([(20, 40)])
|
|
s = np.zeros(100); s[60:70] = 1.0
|
|
f1, _, _ = point_f1(s, y)
|
|
assert f1 == pytest.approx(0.0, abs=1e-6)
|
|
|
|
def test_partial(self):
|
|
y = labels([(20, 40)])
|
|
s = np.zeros(100); s[30:41] = 0.9
|
|
f1, prec, rec = point_f1(s, y, threshold=0.5)
|
|
# predicted 11 anomalous all in true region: precision 1, recall 11/21
|
|
assert prec == pytest.approx(1.0, abs=1e-6)
|
|
assert rec == pytest.approx(11 / 21, abs=1e-3)
|
|
|
|
|
|
class TestAUCPR:
|
|
def test_perfect_ranking(self):
|
|
y = labels([(10, 20)])
|
|
s = np.zeros(100); s[10:21] = 1.0
|
|
assert auc_pr(s, y) == pytest.approx(1.0, abs=1e-6)
|
|
|
|
def test_inverse_ranking_low(self):
|
|
y = labels([(10, 20)])
|
|
s = np.ones(100); s[10:21] = 0.0 # anomalies score LOW
|
|
assert auc_pr(s, y) < 0.5
|
|
|
|
|
|
class TestPAF1:
|
|
def test_pa_inflates(self):
|
|
# PA: if ANY predicted point is in the true anomaly, the whole segment
|
|
# counts as caught → much higher F1 than point-wise
|
|
y = labels([(20, 40)])
|
|
s = np.zeros(100); s[40] = 1.0 # single overlap point
|
|
f1_pa, _, _ = point_f1_with_pa(s, y)
|
|
f1_pt, _, _ = point_f1(s, y)
|
|
assert f1_pa > f1_pt
|
|
|
|
|
|
class TestAffiliationF1:
|
|
def test_perfect(self):
|
|
y = labels([(20, 40)])
|
|
s = np.zeros(100); s[20:41] = 1.0
|
|
f1, _, _ = affiliation_f1(s, y)
|
|
assert f1 == pytest.approx(1.0, abs=1e-6)
|
|
|
|
def test_far_predictions_lower(self):
|
|
y = labels([(20, 40)])
|
|
s_near = np.zeros(100); s_near[41:45] = 1.0
|
|
s_far = np.zeros(100); s_far[90:95] = 1.0
|
|
f1_near, _, _ = affiliation_f1(s_near, y)
|
|
f1_far, _, _ = affiliation_f1(s_far, y)
|
|
assert f1_near > f1_far
|
|
|
|
|
|
class TestVUSPR:
|
|
def test_perfect_high(self):
|
|
y = labels([(20, 40)])
|
|
s = np.zeros(100); s[20:41] = 1.0
|
|
v = vus_pr(s, y)
|
|
assert v == pytest.approx(1.0, abs=1e-3)
|
|
|
|
def test_random_scores_moderate(self):
|
|
rng = np.random.default_rng(0)
|
|
y = labels([(20, 40)])
|
|
s = rng.random(100)
|
|
v = vus_pr(s, y)
|
|
# random ranking → VUS-PR around the anomaly prevalence (~0.21)
|
|
assert 0.0 < v < 0.6
|
|
|
|
def test_all_report_not_inflated(self):
|
|
# spec REQUIREMENT: "全报异常" trivial baseline must NOT have inflated VUS-PR
|
|
y = labels([(20, 40)], T=100)
|
|
s = np.ones(100) # report everything as anomalous
|
|
v = vus_pr(s, y)
|
|
# prevalence = 21/100; all-ones gives AUC-PR == prevalence everywhere
|
|
assert v == pytest.approx(21 / 100, abs=1e-3)
|
|
|
|
def test_returns_scalar(self):
|
|
y = labels([(5, 10)])
|
|
s = np.random.default_rng(1).random(100)
|
|
v = vus_pr(s, y)
|
|
assert np.isscalar(v) or v.shape == ()
|