张宗平
4c4a8f6435
- src/tsmm/eval/parse_answer.py: text→JSON segments→point scores (JSON-first, regex fallback, all-zero on fail). - src/tsmm/eval/ts_metrics.py: VUS-PR (main, threshold-free; constant-score → prevalence, NOT inflated), AUC-PR, Point-F1 (no PA), PA-F1 (control only), self-contained Affiliation-F1. - src/tsmm/eval/qa_judge.py: RuleJudge (anomaly IoU / describe+forecast numeric tolerance) + LLMJudge (stub backend = deterministic offline heuristic for the no-network host; openai backend = GPT-4o 0-5; pluggable callable). - src/tsmm/eval/baselines.py: trivial (Random/Constant/AllReport) + ts_to_text + pure_llm_answer glue; Time-LLM/ChatTS marked not-reproduced (honest). - src/tsmm/eval/report.py + scripts/run_eval.sh: trained model + all baselines through the SAME pipeline → reports/eval-YYYYMMDD.md (VUS-PR main, PA-F1 explicitly labelled control, trivial baselines included). - tests: 45 new (parse 13, metrics 12, judge 12, baselines 8). 145 passing. - Smoke: run_eval on 40 samples produces valid markdown; PA-F1≈1.0 vs VUS-PR≈0.35 for all_report demonstrates why PA must not be the headline.
63 lines
1.8 KiB
Python
63 lines
1.8 KiB
Python
"""Tests for eval/baselines.py (T5.4)."""
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from tsmm.eval.baselines import (
|
|
RandomBaseline,
|
|
ConstantBaseline,
|
|
AllReportBaseline,
|
|
ts_to_text,
|
|
)
|
|
|
|
|
|
class TestTrivial:
|
|
def test_random_in_range_and_shape(self):
|
|
bl = RandomBaseline(seed=0)
|
|
scores = bl.score(T=100)
|
|
assert scores.shape == (100,)
|
|
assert scores.min() >= 0.0 and scores.max() <= 1.0
|
|
|
|
def test_random_reproducible(self):
|
|
a = RandomBaseline(seed=42).score(T=50)
|
|
b = RandomBaseline(seed=42).score(T=50)
|
|
assert np.allclose(a, b)
|
|
|
|
def test_constant_value(self):
|
|
bl = ConstantBaseline(value=0.3)
|
|
s = bl.score(T=10)
|
|
assert s.shape == (10,)
|
|
assert np.allclose(s, 0.3)
|
|
|
|
def test_all_report_is_all_ones(self):
|
|
bl = AllReportBaseline()
|
|
s = bl.score(T=8)
|
|
assert s.shape == (8,)
|
|
assert np.allclose(s, 1.0)
|
|
|
|
def test_all_report_vus_pr_not_inflated(self):
|
|
from tsmm.eval.ts_metrics import vus_pr
|
|
y = np.zeros(100, dtype=int); y[20:40] = 1
|
|
s = AllReportBaseline().score(T=100)
|
|
v = vus_pr(s, y)
|
|
assert v == pytest.approx(20 / 100, abs=1e-3)
|
|
|
|
|
|
class TestTsToText:
|
|
def test_shape_and_content(self):
|
|
series = np.arange(12).reshape(4, 3).tolist() # T=4, C=3
|
|
text = ts_to_text(series, max_points=12)
|
|
assert isinstance(text, str)
|
|
assert "0" in text
|
|
|
|
def test_truncates_long_series(self):
|
|
series = [[float(i)] for i in range(1000)]
|
|
text = ts_to_text(series, max_points=50)
|
|
# roughly bounded length
|
|
assert len(text) < 5000
|
|
|
|
def test_handles_nan(self):
|
|
import math
|
|
series = [[1.0], [float("nan")], [3.0]]
|
|
text = ts_to_text(series, max_points=10)
|
|
assert "nan" in text.lower() or "missing" in text.lower()
|