feat(m5): full eval pipeline — parse/metrics/judge/baselines/report (T5.1-T5.5)
- src/tsmm/eval/parse_answer.py: text→JSON segments→point scores (JSON-first, regex fallback, all-zero on fail). - src/tsmm/eval/ts_metrics.py: VUS-PR (main, threshold-free; constant-score → prevalence, NOT inflated), AUC-PR, Point-F1 (no PA), PA-F1 (control only), self-contained Affiliation-F1. - src/tsmm/eval/qa_judge.py: RuleJudge (anomaly IoU / describe+forecast numeric tolerance) + LLMJudge (stub backend = deterministic offline heuristic for the no-network host; openai backend = GPT-4o 0-5; pluggable callable). - src/tsmm/eval/baselines.py: trivial (Random/Constant/AllReport) + ts_to_text + pure_llm_answer glue; Time-LLM/ChatTS marked not-reproduced (honest). - src/tsmm/eval/report.py + scripts/run_eval.sh: trained model + all baselines through the SAME pipeline → reports/eval-YYYYMMDD.md (VUS-PR main, PA-F1 explicitly labelled control, trivial baselines included). - tests: 45 new (parse 13, metrics 12, judge 12, baselines 8). 145 passing. - Smoke: run_eval on 40 samples produces valid markdown; PA-F1≈1.0 vs VUS-PR≈0.35 for all_report demonstrates why PA must not be the headline.
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
"""Tests for eval/baselines.py (T5.4)."""
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tsmm.eval.baselines import (
|
||||
RandomBaseline,
|
||||
ConstantBaseline,
|
||||
AllReportBaseline,
|
||||
ts_to_text,
|
||||
)
|
||||
|
||||
|
||||
class TestTrivial:
|
||||
def test_random_in_range_and_shape(self):
|
||||
bl = RandomBaseline(seed=0)
|
||||
scores = bl.score(T=100)
|
||||
assert scores.shape == (100,)
|
||||
assert scores.min() >= 0.0 and scores.max() <= 1.0
|
||||
|
||||
def test_random_reproducible(self):
|
||||
a = RandomBaseline(seed=42).score(T=50)
|
||||
b = RandomBaseline(seed=42).score(T=50)
|
||||
assert np.allclose(a, b)
|
||||
|
||||
def test_constant_value(self):
|
||||
bl = ConstantBaseline(value=0.3)
|
||||
s = bl.score(T=10)
|
||||
assert s.shape == (10,)
|
||||
assert np.allclose(s, 0.3)
|
||||
|
||||
def test_all_report_is_all_ones(self):
|
||||
bl = AllReportBaseline()
|
||||
s = bl.score(T=8)
|
||||
assert s.shape == (8,)
|
||||
assert np.allclose(s, 1.0)
|
||||
|
||||
def test_all_report_vus_pr_not_inflated(self):
|
||||
from tsmm.eval.ts_metrics import vus_pr
|
||||
y = np.zeros(100, dtype=int); y[20:40] = 1
|
||||
s = AllReportBaseline().score(T=100)
|
||||
v = vus_pr(s, y)
|
||||
assert v == pytest.approx(20 / 100, abs=1e-3)
|
||||
|
||||
|
||||
class TestTsToText:
|
||||
def test_shape_and_content(self):
|
||||
series = np.arange(12).reshape(4, 3).tolist() # T=4, C=3
|
||||
text = ts_to_text(series, max_points=12)
|
||||
assert isinstance(text, str)
|
||||
assert "0" in text
|
||||
|
||||
def test_truncates_long_series(self):
|
||||
series = [[float(i)] for i in range(1000)]
|
||||
text = ts_to_text(series, max_points=50)
|
||||
# roughly bounded length
|
||||
assert len(text) < 5000
|
||||
|
||||
def test_handles_nan(self):
|
||||
import math
|
||||
series = [[1.0], [float("nan")], [3.0]]
|
||||
text = ts_to_text(series, max_points=10)
|
||||
assert "nan" in text.lower() or "missing" in text.lower()
|
||||
Reference in New Issue
Block a user