feat(m5): full eval pipeline — parse/metrics/judge/baselines/report (T5.1-T5.5)
- src/tsmm/eval/parse_answer.py: text→JSON segments→point scores (JSON-first, regex fallback, all-zero on fail). - src/tsmm/eval/ts_metrics.py: VUS-PR (main, threshold-free; constant-score → prevalence, NOT inflated), AUC-PR, Point-F1 (no PA), PA-F1 (control only), self-contained Affiliation-F1. - src/tsmm/eval/qa_judge.py: RuleJudge (anomaly IoU / describe+forecast numeric tolerance) + LLMJudge (stub backend = deterministic offline heuristic for the no-network host; openai backend = GPT-4o 0-5; pluggable callable). - src/tsmm/eval/baselines.py: trivial (Random/Constant/AllReport) + ts_to_text + pure_llm_answer glue; Time-LLM/ChatTS marked not-reproduced (honest). - src/tsmm/eval/report.py + scripts/run_eval.sh: trained model + all baselines through the SAME pipeline → reports/eval-YYYYMMDD.md (VUS-PR main, PA-F1 explicitly labelled control, trivial baselines included). - tests: 45 new (parse 13, metrics 12, judge 12, baselines 8). 145 passing. - Smoke: run_eval on 40 samples produces valid markdown; PA-F1≈1.0 vs VUS-PR≈0.35 for all_report demonstrates why PA must not be the headline.
This commit is contained in:
@@ -0,0 +1,95 @@
|
||||
"""Tests for eval/parse_answer.py (T5.1)."""
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
|
||||
from tsmm.eval.parse_answer import parse_anomaly_segments, answer_to_point_scores
|
||||
|
||||
|
||||
class TestParseSegments:
|
||||
def test_clean_json(self):
|
||||
text = '{"segments": [[10, 20], [30, 40]], "type": "spike"}'
|
||||
segs, ok = parse_anomaly_segments(text)
|
||||
assert ok is True
|
||||
assert segs == [(10, 20), (30, 40)]
|
||||
|
||||
def test_json_in_prose(self):
|
||||
text = 'The anomalies are: {"segments": [[5, 8]], "type":"level_shift"} per your request.'
|
||||
segs, ok = parse_anomaly_segments(text)
|
||||
assert ok is True
|
||||
assert segs == [(5, 8)]
|
||||
|
||||
def test_nested_brackets(self):
|
||||
text = '{"segments": [[320, 360]], "type":"spike", "score":0.9}'
|
||||
segs, ok = parse_anomaly_segments(text)
|
||||
assert ok is True
|
||||
assert segs == [(320, 360)]
|
||||
|
||||
def test_multiple_json_objects(self):
|
||||
# when multiple JSON-ish blobs exist, take the one with a segments key
|
||||
text = 'some text {"foo": 1} then {"segments":[[1,2]]}'
|
||||
segs, ok = parse_anomaly_segments(text)
|
||||
assert ok is True
|
||||
assert segs == [(1, 2)]
|
||||
|
||||
def test_no_json_returns_empty_fail(self):
|
||||
text = "I cannot find any anomaly."
|
||||
segs, ok = parse_anomaly_segments(text)
|
||||
assert ok is False
|
||||
assert segs == []
|
||||
|
||||
def test_malformed_json_fallback_to_regex(self):
|
||||
# if JSON parse fails but [s,e] patterns exist, regex-extract them
|
||||
text = 'anomalies at [100, 150] and [200, 250]'
|
||||
segs, ok = parse_anomaly_segments(text)
|
||||
# ok=False because no valid JSON, but segments extracted via fallback
|
||||
assert segs == [(100, 150), (200, 250)]
|
||||
|
||||
def test_empty_segments_list(self):
|
||||
text = '{"segments": [], "type": "none"}'
|
||||
segs, ok = parse_anomaly_segments(text)
|
||||
assert ok is True
|
||||
assert segs == []
|
||||
|
||||
def test_single_int_segment(self):
|
||||
# segments might be [start] only or [[s]] — be lenient
|
||||
text = '{"segments": [[42]]}'
|
||||
segs, ok = parse_anomaly_segments(text)
|
||||
assert ok is True
|
||||
# single-element segment treated as point
|
||||
assert segs == [(42, 42)]
|
||||
|
||||
|
||||
class TestAnswerToPointScores:
|
||||
def test_basic(self):
|
||||
text = '{"segments": [[2, 5]]}'
|
||||
scores, ok = answer_to_point_scores(text, T=10)
|
||||
assert ok is True
|
||||
assert scores.shape == (10,)
|
||||
assert list(scores) == [0, 0, 1, 1, 1, 1, 0, 0, 0, 0]
|
||||
|
||||
def test_multiple_segments(self):
|
||||
text = '{"segments": [[0, 2], [7, 9]]}'
|
||||
scores, ok = answer_to_point_scores(text, T=10)
|
||||
assert ok is True
|
||||
assert list(scores) == [1, 1, 1, 0, 0, 0, 0, 1, 1, 1]
|
||||
|
||||
def test_no_anomaly_all_zero(self):
|
||||
text = '{"segments": []}'
|
||||
scores, ok = answer_to_point_scores(text, T=8)
|
||||
assert ok is True
|
||||
assert float(scores.sum()) == 0.0
|
||||
|
||||
def test_parse_fail_all_zero(self):
|
||||
text = "no idea"
|
||||
scores, ok = answer_to_point_scores(text, T=8)
|
||||
assert ok is False
|
||||
assert scores.shape == (8,)
|
||||
assert float(scores.sum()) == 0.0
|
||||
|
||||
def test_clips_to_T(self):
|
||||
# segment beyond T is clipped
|
||||
text = '{"segments": [[5, 100]]}'
|
||||
scores, ok = answer_to_point_scores(text, T=10)
|
||||
assert ok is True
|
||||
assert float(scores.sum()) == 5.0 # indices 5..9
|
||||
Reference in New Issue
Block a user