feat(m5): full eval pipeline — parse/metrics/judge/baselines/report (T5.1-T5.5)

- src/tsmm/eval/parse_answer.py: text→JSON segments→point scores (JSON-first,
  regex fallback, all-zero on fail).
- src/tsmm/eval/ts_metrics.py: VUS-PR (main, threshold-free; constant-score →
  prevalence, NOT inflated), AUC-PR, Point-F1 (no PA), PA-F1 (control only),
  self-contained Affiliation-F1.
- src/tsmm/eval/qa_judge.py: RuleJudge (anomaly IoU / describe+forecast numeric
  tolerance) + LLMJudge (stub backend = deterministic offline heuristic for the
  no-network host; openai backend = GPT-4o 0-5; pluggable callable).
- src/tsmm/eval/baselines.py: trivial (Random/Constant/AllReport) + ts_to_text
  + pure_llm_answer glue; Time-LLM/ChatTS marked not-reproduced (honest).
- src/tsmm/eval/report.py + scripts/run_eval.sh: trained model + all baselines
  through the SAME pipeline → reports/eval-YYYYMMDD.md (VUS-PR main, PA-F1
  explicitly labelled control, trivial baselines included).
- tests: 45 new (parse 13, metrics 12, judge 12, baselines 8). 145 passing.
- Smoke: run_eval on 40 samples produces valid markdown; PA-F1≈1.0 vs
  VUS-PR≈0.35 for all_report demonstrates why PA must not be the headline.

tests/test_baselines.py

@@ -0,0 +1,62 @@
+"""Tests for eval/baselines.py (T5.4)."""
+import numpy as np
+import pytest
+
+from tsmm.eval.baselines import (
+    RandomBaseline,
+    ConstantBaseline,
+    AllReportBaseline,
+    ts_to_text,
+)
+
+
+class TestTrivial:
+    def test_random_in_range_and_shape(self):
+        bl = RandomBaseline(seed=0)
+        scores = bl.score(T=100)
+        assert scores.shape == (100,)
+        assert scores.min() >= 0.0 and scores.max() <= 1.0
+
+    def test_random_reproducible(self):
+        a = RandomBaseline(seed=42).score(T=50)
+        b = RandomBaseline(seed=42).score(T=50)
+        assert np.allclose(a, b)
+
+    def test_constant_value(self):
+        bl = ConstantBaseline(value=0.3)
+        s = bl.score(T=10)
+        assert s.shape == (10,)
+        assert np.allclose(s, 0.3)
+
+    def test_all_report_is_all_ones(self):
+        bl = AllReportBaseline()
+        s = bl.score(T=8)
+        assert s.shape == (8,)
+        assert np.allclose(s, 1.0)
+
+    def test_all_report_vus_pr_not_inflated(self):
+        from tsmm.eval.ts_metrics import vus_pr
+        y = np.zeros(100, dtype=int); y[20:40] = 1
+        s = AllReportBaseline().score(T=100)
+        v = vus_pr(s, y)
+        assert v == pytest.approx(20 / 100, abs=1e-3)
+
+
+class TestTsToText:
+    def test_shape_and_content(self):
+        series = np.arange(12).reshape(4, 3).tolist()  # T=4, C=3
+        text = ts_to_text(series, max_points=12)
+        assert isinstance(text, str)
+        assert "0" in text
+
+    def test_truncates_long_series(self):
+        series = [[float(i)] for i in range(1000)]
+        text = ts_to_text(series, max_points=50)
+        # roughly bounded length
+        assert len(text) < 5000
+
+    def test_handles_nan(self):
+        import math
+        series = [[1.0], [float("nan")], [3.0]]
+        text = ts_to_text(series, max_points=10)
+        assert "nan" in text.lower() or "missing" in text.lower()

tests/test_parse_answer.py

@@ -0,0 +1,95 @@
+"""Tests for eval/parse_answer.py (T5.1)."""
+import json
+
+import numpy as np
+
+from tsmm.eval.parse_answer import parse_anomaly_segments, answer_to_point_scores
+
+
+class TestParseSegments:
+    def test_clean_json(self):
+        text = '{"segments": [[10, 20], [30, 40]], "type": "spike"}'
+        segs, ok = parse_anomaly_segments(text)
+        assert ok is True
+        assert segs == [(10, 20), (30, 40)]
+
+    def test_json_in_prose(self):
+        text = 'The anomalies are: {"segments": [[5, 8]], "type":"level_shift"} per your request.'
+        segs, ok = parse_anomaly_segments(text)
+        assert ok is True
+        assert segs == [(5, 8)]
+
+    def test_nested_brackets(self):
+        text = '{"segments": [[320, 360]], "type":"spike", "score":0.9}'
+        segs, ok = parse_anomaly_segments(text)
+        assert ok is True
+        assert segs == [(320, 360)]
+
+    def test_multiple_json_objects(self):
+        # when multiple JSON-ish blobs exist, take the one with a segments key
+        text = 'some text {"foo": 1} then {"segments":[[1,2]]}'
+        segs, ok = parse_anomaly_segments(text)
+        assert ok is True
+        assert segs == [(1, 2)]
+
+    def test_no_json_returns_empty_fail(self):
+        text = "I cannot find any anomaly."
+        segs, ok = parse_anomaly_segments(text)
+        assert ok is False
+        assert segs == []
+
+    def test_malformed_json_fallback_to_regex(self):
+        # if JSON parse fails but [s,e] patterns exist, regex-extract them
+        text = 'anomalies at [100, 150] and [200, 250]'
+        segs, ok = parse_anomaly_segments(text)
+        # ok=False because no valid JSON, but segments extracted via fallback
+        assert segs == [(100, 150), (200, 250)]
+
+    def test_empty_segments_list(self):
+        text = '{"segments": [], "type": "none"}'
+        segs, ok = parse_anomaly_segments(text)
+        assert ok is True
+        assert segs == []
+
+    def test_single_int_segment(self):
+        # segments might be [start] only or [[s]] — be lenient
+        text = '{"segments": [[42]]}'
+        segs, ok = parse_anomaly_segments(text)
+        assert ok is True
+        # single-element segment treated as point
+        assert segs == [(42, 42)]
+
+
+class TestAnswerToPointScores:
+    def test_basic(self):
+        text = '{"segments": [[2, 5]]}'
+        scores, ok = answer_to_point_scores(text, T=10)
+        assert ok is True
+        assert scores.shape == (10,)
+        assert list(scores) == [0, 0, 1, 1, 1, 1, 0, 0, 0, 0]
+
+    def test_multiple_segments(self):
+        text = '{"segments": [[0, 2], [7, 9]]}'
+        scores, ok = answer_to_point_scores(text, T=10)
+        assert ok is True
+        assert list(scores) == [1, 1, 1, 0, 0, 0, 0, 1, 1, 1]
+
+    def test_no_anomaly_all_zero(self):
+        text = '{"segments": []}'
+        scores, ok = answer_to_point_scores(text, T=8)
+        assert ok is True
+        assert float(scores.sum()) == 0.0
+
+    def test_parse_fail_all_zero(self):
+        text = "no idea"
+        scores, ok = answer_to_point_scores(text, T=8)
+        assert ok is False
+        assert scores.shape == (8,)
+        assert float(scores.sum()) == 0.0
+
+    def test_clips_to_T(self):
+        # segment beyond T is clipped
+        text = '{"segments": [[5, 100]]}'
+        scores, ok = answer_to_point_scores(text, T=10)
+        assert ok is True
+        assert float(scores.sum()) == 5.0  # indices 5..9

tests/test_qa_judge.py

@@ -0,0 +1,105 @@
+"""Tests for eval/qa_judge.py (T5.3)."""
+import json
+
+import pytest
+
+from tsmm.eval.qa_judge import (
+    RuleJudge,
+    rule_score_describe,
+    rule_score_anomaly,
+    rule_score_forecast,
+)
+
+
+class TestDescribeRule:
+    def test_numerical_within_tolerance(self):
+        # predicted mean within 5% of reference → 1.0
+        out = rule_score_describe(
+            pred='{"mean": 50.0, "std": 2.0}',
+            ref={"mean": 50.5, "std": 2.1},
+            rtol=0.05,
+        )
+        assert out["score"] == 1.0
+        assert out["parsed"] is True
+
+    def test_outside_tolerance(self):
+        out = rule_score_describe(
+            pred='{"mean": 80.0}',
+            ref={"mean": 50.0},
+            rtol=0.05,
+        )
+        assert out["score"] == 0.0
+        assert out["parsed"] is True
+
+    def test_unparseable(self):
+        out = rule_score_describe(pred="the mean is around 50", ref={"mean": 50.0})
+        assert out["parsed"] is False
+        assert out["score"] == 0.0
+
+
+class TestAnomalyRule:
+    def test_exact_match(self):
+        pred = '{"segments": [[10, 20]]}'
+        ref_segs = [(10, 20)]
+        out = rule_score_anomaly(pred, ref_segs, T=50)
+        assert out["score"] == 1.0
+        assert out["parsed"] is True
+
+    def test_iou_partial(self):
+        pred = '{"segments": [[10, 30]]}'
+        ref_segs = [(20, 40)]
+        out = rule_score_anomaly(pred, ref_segs, T=50)
+        assert 0.0 < out["score"] < 1.0
+
+    def test_no_overlap(self):
+        pred = '{"segments": [[0, 5]]}'
+        ref_segs = [(20, 40)]
+        out = rule_score_anomaly(pred, ref_segs, T=50)
+        assert out["score"] == 0.0
+
+
+class TestForecastRule:
+    def test_within_tolerance(self):
+        pred = '{"forecast": [1.0, 2.0, 3.0]}'
+        ref = [1.05, 1.95, 3.1]
+        out = rule_score_forecast(pred, ref, rtol=0.1)
+        assert out["score"] == 1.0
+        assert out["parsed"] is True
+
+    def test_one_off(self):
+        pred = '{"forecast": [1.0, 2.0, 9.0]}'
+        ref = [1.0, 2.0, 3.0]
+        out = rule_score_forecast(pred, ref, rtol=0.1)
+        assert out["score"] < 1.0
+
+
+class TestRuleJudgeDispatch:
+    def test_judge_anomaly(self):
+        judge = RuleJudge()
+        out = judge.judge("anomaly", '{"segments": [[5, 8]]}', ref_segs=[(5, 8)], T=20)
+        assert out["score"] == 1.0
+        assert out["track"] == "rule"
+
+    def test_judge_unknown_category_skips(self):
+        judge = RuleJudge()
+        out = judge.judge("root_cause", "some text answer", ref_answer="some text answer")
+        assert out["track"] == "skip"
+        assert out["score"] is None
+
+
+class TestLLMJudgeStub:
+    def test_stub_returns_deterministic(self):
+        from tsmm.eval.qa_judge import LLMJudge
+        judge = LLMJudge(backend="stub")
+        s1 = judge.score("Q?", "ref", "pred")
+        s2 = judge.score("Q?", "ref", "pred")
+        assert isinstance(s1, dict)
+        assert "score" in s1 and 0 <= s1["score"] <= 5
+        assert s1 == s2  # deterministic stub
+
+    def test_stub_rewards_exact_match(self):
+        from tsmm.eval.qa_judge import LLMJudge
+        judge = LLMJudge(backend="stub")
+        same = judge.score("Q?", "正常", "正常")["score"]
+        diff = judge.score("Q?", "正常", "完全不同的乱码回答xyz")["score"]
+        assert same >= diff

tests/test_ts_metrics.py

@@ -0,0 +1,109 @@
+"""Tests for eval/ts_metrics.py (T5.2)."""
+import numpy as np
+import pytest
+
+from tsmm.eval.ts_metrics import (
+    affiliation_f1,
+    auc_pr,
+    point_f1,
+    point_f1_with_pa,
+    vus_pr,
+)
+
+
+def labels(segs, T=100):
+    y = np.zeros(T, dtype=int)
+    for a, b in segs:
+        y[a:b + 1] = 1
+    return y
+
+
+class TestPointF1:
+    def test_perfect(self):
+        y = labels([(20, 40)], T=100)
+        s = np.zeros(100); s[20:41] = 1.0
+        f1, prec, rec = point_f1(s, y)
+        assert f1 == pytest.approx(1.0, abs=1e-6)
+
+    def test_no_overlap(self):
+        y = labels([(20, 40)])
+        s = np.zeros(100); s[60:70] = 1.0
+        f1, _, _ = point_f1(s, y)
+        assert f1 == pytest.approx(0.0, abs=1e-6)
+
+    def test_partial(self):
+        y = labels([(20, 40)])
+        s = np.zeros(100); s[30:41] = 0.9
+        f1, prec, rec = point_f1(s, y, threshold=0.5)
+        # predicted 11 anomalous all in true region: precision 1, recall 11/21
+        assert prec == pytest.approx(1.0, abs=1e-6)
+        assert rec == pytest.approx(11 / 21, abs=1e-3)
+
+
+class TestAUCPR:
+    def test_perfect_ranking(self):
+        y = labels([(10, 20)])
+        s = np.zeros(100); s[10:21] = 1.0
+        assert auc_pr(s, y) == pytest.approx(1.0, abs=1e-6)
+
+    def test_inverse_ranking_low(self):
+        y = labels([(10, 20)])
+        s = np.ones(100); s[10:21] = 0.0  # anomalies score LOW
+        assert auc_pr(s, y) < 0.5
+
+
+class TestPAF1:
+    def test_pa_inflates(self):
+        # PA: if ANY predicted point is in the true anomaly, the whole segment
+        # counts as caught → much higher F1 than point-wise
+        y = labels([(20, 40)])
+        s = np.zeros(100); s[40] = 1.0  # single overlap point
+        f1_pa, _, _ = point_f1_with_pa(s, y)
+        f1_pt, _, _ = point_f1(s, y)
+        assert f1_pa > f1_pt
+
+
+class TestAffiliationF1:
+    def test_perfect(self):
+        y = labels([(20, 40)])
+        s = np.zeros(100); s[20:41] = 1.0
+        f1, _, _ = affiliation_f1(s, y)
+        assert f1 == pytest.approx(1.0, abs=1e-6)
+
+    def test_far_predictions_lower(self):
+        y = labels([(20, 40)])
+        s_near = np.zeros(100); s_near[41:45] = 1.0
+        s_far = np.zeros(100); s_far[90:95] = 1.0
+        f1_near, _, _ = affiliation_f1(s_near, y)
+        f1_far, _, _ = affiliation_f1(s_far, y)
+        assert f1_near > f1_far
+
+
+class TestVUSPR:
+    def test_perfect_high(self):
+        y = labels([(20, 40)])
+        s = np.zeros(100); s[20:41] = 1.0
+        v = vus_pr(s, y)
+        assert v == pytest.approx(1.0, abs=1e-3)
+
+    def test_random_scores_moderate(self):
+        rng = np.random.default_rng(0)
+        y = labels([(20, 40)])
+        s = rng.random(100)
+        v = vus_pr(s, y)
+        # random ranking → VUS-PR around the anomaly prevalence (~0.21)
+        assert 0.0 < v < 0.6
+
+    def test_all_report_not_inflated(self):
+        # spec REQUIREMENT: "全报异常" trivial baseline must NOT have inflated VUS-PR
+        y = labels([(20, 40)], T=100)
+        s = np.ones(100)  # report everything as anomalous
+        v = vus_pr(s, y)
+        # prevalence = 21/100; all-ones gives AUC-PR == prevalence everywhere
+        assert v == pytest.approx(21 / 100, abs=1e-3)
+
+    def test_returns_scalar(self):
+        y = labels([(5, 10)])
+        s = np.random.default_rng(1).random(100)
+        v = vus_pr(s, y)
+        assert np.isscalar(v) or v.shape == ()