fix(eval): parse_anomaly_segments must tolerate non-numeric junk entries
Full-scale eval crashed in the pure-LLM QA-judge path: pure_llm emits
free-form JSON whose 'segments' list can contain entries like
["CPU Usage", 30] (channel name instead of an int). _seg_from_entry called
int() unconditionally and raised ValueError, killing the whole report run
(model predictor had already finished; result lost).
Fix: wrap _seg_from_entry in try/except (ValueError, TypeError) → return
None for any unparseable entry, so good segments alongside junk are still
extracted. Verified: {["CPU Usage",30],[10,20]} → [(10,20)]. + unit test.
Also archive logs/train_full_run.log (stage1 31250 EMA1.26, stage2 9375 EMA0.75).
This commit is contained in:
@@ -79,8 +79,10 @@ def _seg_from_entry(s):
|
|||||||
|
|
||||||
Accepts either ``[start, end]`` lists/tuples or ``{"start": s, "end": e}``
|
Accepts either ``[start, end]`` lists/tuples or ``{"start": s, "end": e}``
|
||||||
objects (the schema emitted by ``data/instruct.py``). Returns None if no
|
objects (the schema emitted by ``data/instruct.py``). Returns None if no
|
||||||
usable pair is found.
|
usable pair is found, or if the values aren't ints (e.g. a pure-LLM emits
|
||||||
|
``["CPU Usage", 30]`` — junk that must not crash the eval pipeline).
|
||||||
"""
|
"""
|
||||||
|
try:
|
||||||
if isinstance(s, dict):
|
if isinstance(s, dict):
|
||||||
if "start" in s and "end" in s:
|
if "start" in s and "end" in s:
|
||||||
return (int(s["start"]), int(s["end"]))
|
return (int(s["start"]), int(s["end"]))
|
||||||
@@ -90,6 +92,9 @@ def _seg_from_entry(s):
|
|||||||
return _normalize_seg(s)
|
return _normalize_seg(s)
|
||||||
if len(s) == 1:
|
if len(s) == 1:
|
||||||
v = int(s[0]); return (v, v)
|
v = int(s[0]); return (v, v)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return None
|
||||||
|
return None
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -73,6 +73,18 @@ class TestParseSegments:
|
|||||||
assert any('"a"' in b and '"b"' in b for b in blobs)
|
assert any('"a"' in b and '"b"' in b for b in blobs)
|
||||||
assert '{"d": 4}' in blobs
|
assert '{"d": 4}' in blobs
|
||||||
|
|
||||||
|
def test_non_numeric_segment_entries_are_skipped(self):
|
||||||
|
# a pure-LLM may emit junk like ["CPU Usage", 30] — must not crash, just skip
|
||||||
|
from tsmm.eval.parse_answer import _seg_from_entry
|
||||||
|
assert _seg_from_entry(["CPU Usage", 30]) is None
|
||||||
|
assert _seg_from_entry({"start": "foo", "end": "bar"}) is None
|
||||||
|
assert _seg_from_entry("not a segment") is None
|
||||||
|
# good entries alongside junk are still extracted
|
||||||
|
text = '{"segments": [["CPU Usage", 30], [10, 20]]}'
|
||||||
|
segs, ok = parse_anomaly_segments(text)
|
||||||
|
assert ok is True
|
||||||
|
assert segs == [(10, 20)]
|
||||||
|
|
||||||
def test_empty_segments_list(self):
|
def test_empty_segments_list(self):
|
||||||
text = '{"segments": [], "type": "none"}'
|
text = '{"segments": [], "type": "none"}'
|
||||||
segs, ok = parse_anomaly_segments(text)
|
segs, ok = parse_anomaly_segments(text)
|
||||||
|
|||||||
Reference in New Issue
Block a user