张宗平
17ed7947d3
Full-scale eval crashed in the pure-LLM QA-judge path: pure_llm emits
free-form JSON whose 'segments' list can contain entries like
["CPU Usage", 30] (channel name instead of an int). _seg_from_entry called
int() unconditionally and raised ValueError, killing the whole report run
(model predictor had already finished; result lost).
Fix: wrap _seg_from_entry in try/except (ValueError, TypeError) → return
None for any unparseable entry, so good segments alongside junk are still
extracted. Verified: {["CPU Usage",30],[10,20]} → [(10,20)]. + unit test.
Also archive logs/train_full_run.log (stage1 31250 EMA1.26, stage2 9375 EMA0.75).
136 lines
5.2 KiB
Python
136 lines
5.2 KiB
Python
"""Tests for eval/parse_answer.py (T5.1)."""
|
|
import json
|
|
|
|
import numpy as np
|
|
|
|
from tsmm.eval.parse_answer import parse_anomaly_segments, answer_to_point_scores
|
|
|
|
|
|
class TestParseSegments:
|
|
def test_clean_json(self):
|
|
text = '{"segments": [[10, 20], [30, 40]], "type": "spike"}'
|
|
segs, ok = parse_anomaly_segments(text)
|
|
assert ok is True
|
|
assert segs == [(10, 20), (30, 40)]
|
|
|
|
def test_json_in_prose(self):
|
|
text = 'The anomalies are: {"segments": [[5, 8]], "type":"level_shift"} per your request.'
|
|
segs, ok = parse_anomaly_segments(text)
|
|
assert ok is True
|
|
assert segs == [(5, 8)]
|
|
|
|
def test_nested_brackets(self):
|
|
text = '{"segments": [[320, 360]], "type":"spike", "score":0.9}'
|
|
segs, ok = parse_anomaly_segments(text)
|
|
assert ok is True
|
|
assert segs == [(320, 360)]
|
|
|
|
def test_multiple_json_objects(self):
|
|
# when multiple JSON-ish blobs exist, take the one with a segments key
|
|
text = 'some text {"foo": 1} then {"segments":[[1,2]]}'
|
|
segs, ok = parse_anomaly_segments(text)
|
|
assert ok is True
|
|
assert segs == [(1, 2)]
|
|
|
|
def test_no_json_returns_empty_fail(self):
|
|
text = "I cannot find any anomaly."
|
|
segs, ok = parse_anomaly_segments(text)
|
|
assert ok is False
|
|
assert segs == []
|
|
|
|
def test_malformed_json_fallback_to_regex(self):
|
|
# if JSON parse fails but [s,e] patterns exist, regex-extract them
|
|
text = 'anomalies at [100, 150] and [200, 250]'
|
|
segs, ok = parse_anomaly_segments(text)
|
|
# ok=False because no valid JSON, but segments extracted via fallback
|
|
assert segs == [(100, 150), (200, 250)]
|
|
|
|
def test_anomaly_regions_schema(self):
|
|
# the schema data/instruct.py actually emits (nested, start/end objects)
|
|
text = '{"anomaly_regions": [{"start": 10, "end": 93}, {"start": 245, "end": 286}], "n_regions": 2}'
|
|
segs, ok = parse_anomaly_segments(text)
|
|
assert ok is True
|
|
assert segs == [(10, 93), (245, 286)]
|
|
|
|
def test_anomaly_regions_in_prose(self):
|
|
text = 'Based on the series: {"anomaly_regions": [{"start": 5, "end": 8}]} done.'
|
|
segs, ok = parse_anomaly_segments(text)
|
|
assert ok is True
|
|
assert segs == [(5, 8)]
|
|
|
|
def test_segments_with_start_end_objects(self):
|
|
# also accept the {start,end} object form under the 'segments' key
|
|
text = '{"segments": [{"start": 1, "end": 4}]}'
|
|
segs, ok = parse_anomaly_segments(text)
|
|
assert ok is True
|
|
assert segs == [(1, 4)]
|
|
|
|
def test_balanced_json_handles_nested(self):
|
|
from tsmm.eval.parse_answer import _extract_balanced_json
|
|
text = 'x {"a": {"b": 1}, "c": [2,3]} y {"d": 4}'
|
|
blobs = _extract_balanced_json(text)
|
|
# outer nested object + the flat one, but NOT the inner {"b":1} alone
|
|
assert any('"a"' in b and '"b"' in b for b in blobs)
|
|
assert '{"d": 4}' in blobs
|
|
|
|
def test_non_numeric_segment_entries_are_skipped(self):
|
|
# a pure-LLM may emit junk like ["CPU Usage", 30] — must not crash, just skip
|
|
from tsmm.eval.parse_answer import _seg_from_entry
|
|
assert _seg_from_entry(["CPU Usage", 30]) is None
|
|
assert _seg_from_entry({"start": "foo", "end": "bar"}) is None
|
|
assert _seg_from_entry("not a segment") is None
|
|
# good entries alongside junk are still extracted
|
|
text = '{"segments": [["CPU Usage", 30], [10, 20]]}'
|
|
segs, ok = parse_anomaly_segments(text)
|
|
assert ok is True
|
|
assert segs == [(10, 20)]
|
|
|
|
def test_empty_segments_list(self):
|
|
text = '{"segments": [], "type": "none"}'
|
|
segs, ok = parse_anomaly_segments(text)
|
|
assert ok is True
|
|
assert segs == []
|
|
|
|
def test_single_int_segment(self):
|
|
# segments might be [start] only or [[s]] — be lenient
|
|
text = '{"segments": [[42]]}'
|
|
segs, ok = parse_anomaly_segments(text)
|
|
assert ok is True
|
|
# single-element segment treated as point
|
|
assert segs == [(42, 42)]
|
|
|
|
|
|
class TestAnswerToPointScores:
|
|
def test_basic(self):
|
|
text = '{"segments": [[2, 5]]}'
|
|
scores, ok = answer_to_point_scores(text, T=10)
|
|
assert ok is True
|
|
assert scores.shape == (10,)
|
|
assert list(scores) == [0, 0, 1, 1, 1, 1, 0, 0, 0, 0]
|
|
|
|
def test_multiple_segments(self):
|
|
text = '{"segments": [[0, 2], [7, 9]]}'
|
|
scores, ok = answer_to_point_scores(text, T=10)
|
|
assert ok is True
|
|
assert list(scores) == [1, 1, 1, 0, 0, 0, 0, 1, 1, 1]
|
|
|
|
def test_no_anomaly_all_zero(self):
|
|
text = '{"segments": []}'
|
|
scores, ok = answer_to_point_scores(text, T=8)
|
|
assert ok is True
|
|
assert float(scores.sum()) == 0.0
|
|
|
|
def test_parse_fail_all_zero(self):
|
|
text = "no idea"
|
|
scores, ok = answer_to_point_scores(text, T=8)
|
|
assert ok is False
|
|
assert scores.shape == (8,)
|
|
assert float(scores.sum()) == 0.0
|
|
|
|
def test_clips_to_T(self):
|
|
# segment beyond T is clipped
|
|
text = '{"segments": [[5, 100]]}'
|
|
scores, ok = answer_to_point_scores(text, T=10)
|
|
assert ok is True
|
|
assert float(scores.sum()) == 5.0 # indices 5..9
|