fix(m5): eval pipeline — anomaly_regions schema + nested JSON + pure-LLM baseline

Critical eval-pipeline bugs found while running the stage-2 model:

1. parse_anomaly_segments only recognized {"segments\:[[s,e]]}; the training
   data (data/instruct.py) emits {"anomaly_regions":[{"start","end"}]}.
   The model learned the correct format but scored 0 → VUS-PR=0. Now accepts
   both schemas + a balanced-brace JSON scanner (the flat regex could not
   match nested objects the model emits).
2. qa_judge._extract_json had the same nesting bug — reuse the scanner.
3. Generation truncated at 64-96 tokens (too short for stats JSON) → bumped
   to 192 in report + instruct_check.

Report enhancements:
- wire the pure-LLM baseline (series-as-text, no TS modality) so design §5.4
  'model vs pure LLM' is actually computed. Heavy predictors built/evaluated
  one at a time with GPU cleanup (single 12GB card).
- add QA-judge table across all 6 categories (design §6.5 问答均分).
- add --pure_llm / --max_per_cat flags.

Results (reports/eval-2026-06-30.md): model QA-judge mean 2.45 vs pure_llm
0.88 (2.8x); anomaly localization (VUS-PR) is the weak point — neither
model nor pure_llm localize well; TS-modality ablation change_rate 0.98.

src/tsmm/eval/instruct_check.py

@@ -76,7 +76,7 @@ def _generate(m, col, sample) -> str:
     import torch
     batch = col([sample])
     with torch.no_grad():
-        txt = m.generate(batch, max_new_tokens=96)
+        txt = m.generate(batch, max_new_tokens=192)
     return txt[0] if txt else ""
 
 
@@ -115,12 +115,21 @@ def evaluate(samples_by_cat, model, col, judge_backend="stub") -> Dict[str, Dict
             if pred.strip():
                 nonempty += 1
 
+            # describe expects a dict ref; parse JSON if possible, else {}
+            ref_for_describe = ref
+            if cat == "describe":
+                try:
+                    ref_for_describe = json.loads(ref) if isinstance(ref, str) else (ref or {})
+                    if not isinstance(ref_for_describe, dict):
+                        ref_for_describe = {}
+                except (ValueError, TypeError):
+                    ref_for_describe = {}
             r = rule_j.judge(
                 cat, pred,
                 ref_segs=[(seg.get("start", 0), seg.get("end", 0))
                           for seg in s.get("segments", [])],
                 T=len(s.get("labels", [])) or 1,
-                ref_answer=ref if cat == "describe" else "",
+                ref_answer=ref_for_describe,
                 ref_forecast=_forecast_list(ref) if cat == "forecast" else [],
             )
             if r.get("score") is not None:
@@ -155,6 +164,8 @@ def _forecast_list(ref_answer):
     """Extract the forecast value list from a forecast ref answer JSON."""
     try:
         obj = json.loads(ref_answer) if isinstance(ref_answer, str) else ref_answer
+        if not isinstance(obj, dict):
+            return []
         fc = obj.get("forecast")
         if isinstance(fc, list) and fc:
             inner = fc[0]

src/tsmm/eval/parse_answer.py

@@ -21,10 +21,52 @@ import numpy as np
 
 # match [int, int] or [[int, int]] style intervals
 _PAIR_RE = re.compile(r"\[\s*(\d+)\s*,\s*(\d+)\s*\]")
-# find JSON-ish objects in text
+# find JSON-ish objects in text (flat blobs only; nested handled by scanner)
 _JSON_RE = re.compile(r"\{[^{}]*\}", re.DOTALL)
 
 
+def _extract_balanced_json(text: str) -> List[str]:
+    """Yield every balanced ``{...}`` substring (handles nesting).
+
+    Scans the text for ``{`` and tracks depth, emitting the span for each
+    top-level balanced object. This lets us parse nested JSON the model emits
+    (e.g. ``{"anomaly_regions": [{"start": 10, "end": 93}]}``) which the
+    flat ``_JSON_RE`` cannot match.
+    """
+    out = []
+    depth = 0
+    start = -1
+    instr = False
+    esc = False
+    quote = ""
+    for i, ch in enumerate(text):
+        if esc:
+            esc = False
+            continue
+        if ch == "\\":
+            esc = True
+            continue
+        if instr:
+            if ch == quote:
+                instr = False
+            continue
+        if ch in ("'", '"'):
+            instr = True
+            quote = ch
+            continue
+        if ch == "{":
+            if depth == 0:
+                start = i
+            depth += 1
+        elif ch == "}":
+            if depth > 0:
+                depth -= 1
+                if depth == 0 and start >= 0:
+                    out.append(text[start : i + 1])
+                    start = -1
+    return out
+
+
 def _normalize_seg(pair) -> Tuple[int, int]:
     a, b = int(pair[0]), int(pair[1])
     if a > b:
@@ -32,36 +74,59 @@ def _normalize_seg(pair) -> Tuple[int, int]:
     return (a, b)
 
 
+def _seg_from_entry(s):
+    """Extract a (start, end) tuple from one segment entry.
+
+    Accepts either ``[start, end]`` lists/tuples or ``{"start": s, "end": e}``
+    objects (the schema emitted by ``data/instruct.py``). Returns None if no
+    usable pair is found.
+    """
+    if isinstance(s, dict):
+        if "start" in s and "end" in s:
+            return (int(s["start"]), int(s["end"]))
+        return None
+    if isinstance(s, (list, tuple)):
+        if len(s) >= 2:
+            return _normalize_seg(s)
+        if len(s) == 1:
+            v = int(s[0]); return (v, v)
+    return None
+
+
 def parse_anomaly_segments(text: str) -> Tuple[List[Tuple[int, int]], bool]:
-    """Return (segments, json_ok). json_ok=True iff a JSON ``segments`` key found."""
+    """Return (segments, json_ok).
+
+    json_ok=True iff a JSON object with a recognized segments key was found.
+    Recognized keys: ``segments`` (list of [s,e]) and ``anomaly_regions``
+    (list of {"start":s,"end":e} — the schema used by ``data/instruct.py``).
+    """
     if text is None:
         return [], False
-    # 1. try every {...} blob; prefer one with a 'segments' key
-    candidates = _JSON_RE.findall(text)
+    # 1. try every balanced {...} blob; prefer one with a recognized segments key
+    candidates = _extract_balanced_json(text)
     best_segments = None
     for blob in candidates:
         try:
             obj = json.loads(blob)
         except json.JSONDecodeError:
             continue
-        if isinstance(obj, dict) and "segments" in obj:
-            segs = obj["segments"]
-            if not isinstance(segs, list):
-                continue
-            out = []
-            for s in segs:
-                if isinstance(s, (list, tuple)):
-                    if len(s) >= 2:
-                        out.append(_normalize_seg(s))
-                    elif len(s) == 1:
-                        v = int(s[0]); out.append((v, v))
-                # ignore non-list entries
-            best_segments = out
-            break
+        if not isinstance(obj, dict):
+            continue
+        raw = None
+        if "segments" in obj:
+            raw = obj["segments"]
+        elif "anomaly_regions" in obj:
+            raw = obj["anomaly_regions"]
+        if raw is None or not isinstance(raw, list):
+            continue
+        out = [_seg_from_entry(s) for s in raw]
+        out = [seg for seg in out if seg is not None]
+        best_segments = out
+        break
     if best_segments is not None:
         return best_segments, True
 
-    # 2. fallback: regex-extract [int,int] pairs (no valid JSON segments found)
+    # 2. fallback: regex-extract [int,int] pairs (no valid JSON key found)
     pairs = _PAIR_RE.findall(text)
     if pairs:
         return [_normalize_seg(p) for p in pairs], False

src/tsmm/eval/qa_judge.py

@@ -16,7 +16,7 @@ import json
 import re
 from typing import Any, Callable, Dict, List, Optional, Tuple
 
-from .parse_answer import parse_anomaly_segments
+from .parse_answer import _extract_balanced_json, parse_anomaly_segments
 
 
 _JSON_RE = re.compile(r"\{[^{}]*\}", re.DOTALL)
@@ -25,6 +25,15 @@ _JSON_RE = re.compile(r"\{[^{}]*\}", re.DOTALL)
 def _extract_json(pred: str) -> Optional[dict]:
     if pred is None:
         return None
+    # prefer balanced (nested-capable) extraction first
+    for blob in _extract_balanced_json(pred):
+        try:
+            obj = json.loads(blob)
+            if isinstance(obj, dict):
+                return obj
+        except json.JSONDecodeError:
+            continue
+    # fallback to flat blobs
     for blob in _JSON_RE.findall(pred):
         try:
             return json.loads(blob)

src/tsmm/eval/report.py

@@ -157,63 +157,121 @@ def run_eval(args: argparse.Namespace) -> str:
     if args.real and os.path.exists(args.real):
         datasets.append(("real", args.real))
 
-    # predictors: trained model (optional) + trivial baselines
+    # predictors: trained model (optional) + trivial baselines + pure-LLM
     predictors: Dict[str, Any] = {
         "random": trivial_predictor(RandomBaseline(seed=0)),
         "constant": trivial_predictor(ConstantBaseline(0.5)),
         "all_report": trivial_predictor(AllReportBaseline()),
     }
+
+    # Heavy predictors are built & evaluated one at a time with GPU cleanup so
+    # we never hold two full models on a single 12GB card.
+    heavy_builders: Dict[str, Any] = {}  # name -> zero-arg callable returning predictor fn
     if args.model:
-        # lazy heavy import
-        import torch
-        from ..model.ts_encoder import TSEncoder
-        from ..model.projector import Projector
-        from ..model.wrapper import MultimodalTSModel
-        from ..data.collator import Collator
+        def _build_trained():
+            import torch
+            from ..model.ts_encoder import TSEncoder
+            from ..model.projector import Projector
+            from ..model.wrapper import MultimodalTSModel
+            from ..data.collator import Collator
 
-        LLM = os.environ.get("TSMM_LLM_PATH", "/home/zhangzp/models/Qwen2.5-0.5B-Instruct")
-        m = MultimodalTSModel(LLM, TSEncoder(), Projector(), dtype=torch.bfloat16).cuda()
-        if args.stage2_ckpt and os.path.exists(args.stage2_ckpt):
-            st = torch.load(args.stage2_ckpt, map_location="cpu")
-            m.encoder.load_state_dict(st["encoder"])
-            m.projector.load_state_dict(st["projector"])
-            lora_dir = os.path.join(os.path.dirname(args.stage2_ckpt), "lora_adapter")
-            if os.path.exists(lora_dir):
-                from peft import PeftModel
-                m.llm = PeftModel.from_pretrained(m.llm, lora_dir)
-        elif args.stage1_ckpt and os.path.exists(args.stage1_ckpt):
-            st = torch.load(args.stage1_ckpt, map_location="cpu")
-            m.encoder.load_state_dict(st["encoder"])
-            m.projector.load_state_dict(st["projector"])
-        m.eval()
-        col = Collator(max_T=512)
+            LLM = os.environ.get("TSMM_LLM_PATH", "/home/zhangzp/models/Qwen2.5-0.5B-Instruct")
+            m = MultimodalTSModel(LLM, TSEncoder(), Projector(), dtype=torch.bfloat16).cuda()
+            if args.stage2_ckpt and os.path.exists(args.stage2_ckpt):
+                st = torch.load(args.stage2_ckpt, map_location="cpu")
+                m.encoder.load_state_dict(st["encoder"])
+                m.projector.load_state_dict(st["projector"])
+                lora_dir = os.path.join(os.path.dirname(args.stage2_ckpt), "lora_adapter")
+                if os.path.exists(lora_dir):
+                    from peft import PeftModel
+                    m.llm = PeftModel.from_pretrained(m.llm, lora_dir)
+            elif args.stage1_ckpt and os.path.exists(args.stage1_ckpt):
+                st = torch.load(args.stage1_ckpt, map_location="cpu")
+                m.encoder.load_state_dict(st["encoder"])
+                m.projector.load_state_dict(st["projector"])
+            m.eval()
+            col = Collator(max_T=512)
 
-        def _model_pred(sample, _m=m, _col=col):
-            batch = _col([sample])
-            with torch.no_grad():
-                txt = _m.generate(batch, max_new_tokens=64)
-            return txt[0] if txt else ""
+            def _pred(sample, _m=m, _col=col):
+                batch = _col([sample])
+                with torch.no_grad():
+                    txt = _m.generate(batch, max_new_tokens=192)
+                return txt[0] if txt else ""
+            return _pred
 
-        predictors = {"model": _model_pred, **predictors}
+        def _build_pure_llm():
+            # base LLM, series fed as TEXT (no TS modality, no LoRA) — design §5.4
+            import torch
+            from ..model.ts_encoder import TSEncoder
+            from ..model.projector import Projector
+            from ..model.wrapper import MultimodalTSModel
+            LLM = os.environ.get("TSMM_LLM_PATH", "/home/zhangzp/models/Qwen2.5-0.5B-Instruct")
+            m = MultimodalTSModel(LLM, TSEncoder(), Projector(), dtype=torch.bfloat16).cuda()
+            m.eval()
+            tok = m.tokenizer
+
+            def _pred(sample, _m=m, _tok=tok):
+                from .baselines import pure_llm_answer
+                return pure_llm_answer(
+                    sample.get("series", []),
+                    sample.get("instruction") or sample.get("question") or "",
+                    model=_m, max_new_tokens=192)
+            return _pred
+
+        heavy_builders["model"] = _build_trained
+        if args.pure_llm:
+            heavy_builders["pure_llm"] = _build_pure_llm
 
     # run anomaly eval per dataset × predictor
     results: Dict[str, Dict[str, Any]] = defaultdict(dict)
     qa_results: Dict[str, Dict[str, Dict[str, Any]]] = defaultdict(lambda: defaultdict(dict))
+    # First pass: cache per-dataset splits so heavy predictors reuse them.
+    splits: Dict[str, Dict[str, List[dict]]] = {}
+    maxpc = args.max_per_cat
     for ds_name, path in datasets:
         samples = list(stream_jsonl(path))
         by_cat: Dict[str, List[dict]] = defaultdict(list)
         for s in samples:
             by_cat[s.get("category", "?")].append(s)
-        anom = by_cat.get("anomaly", [])
-        for pname, pred in predictors.items():
+        if maxpc and maxpc > 0:
+            for cat in list(by_cat.keys()):
+                by_cat[cat] = by_cat[cat][:maxpc]
+        splits[ds_name] = by_cat
+
+    def _eval_predictor(pname, pred):
+        for ds_name, by_cat in splits.items():
+            anom = by_cat.get("anomaly", [])
             results[ds_name][pname] = eval_anomaly_task(anom, pred)
-            # QA judge across all categories (covers design §6.5 问答均分)
             for cat, cat_samples in by_cat.items():
                 qa_results[ds_name][pname][cat] = eval_qa_task(
                     cat_samples, pred, judge_backend=args.judge_backend)
 
+    # trivial (CPU) predictors first
+    all_pred_names = list(predictors.keys())
+    for pname, pred in predictors.items():
+        _eval_predictor(pname, pred)
+
+    # heavy predictors one at a time, with GPU cleanup between
+    if heavy_builders:
+        import torch
+        import gc
+        for pname, builder in heavy_builders.items():
+            print(f"[run_eval] building heavy predictor: {pname}")
+            pred = builder()
+            _eval_predictor(pname, pred)
+            all_pred_names.append(pname)
+            # free GPU before the next heavy model
+            del pred
+            gc.collect()
+            torch.cuda.empty_cache()
+
+    predictor_order = [n for n in ["model", "pure_llm"] if n in all_pred_names]
+    for n in all_pred_names:
+        if n not in predictor_order:
+            predictor_order.append(n)
+
     # render markdown
-    return _render_report(results, qa_results, datasets, predictors.keys(), args)
+    return _render_report(results, qa_results, datasets, predictor_order, args)
 
 
 def _render_report(results, qa_results, datasets, predictor_names, args) -> str:
@@ -293,9 +351,13 @@ def main() -> None:
     p.add_argument("--synth", default="data/eval_synth.jsonl")
     p.add_argument("--real", default="data/eval_real.jsonl")
     p.add_argument("--model", action="store_true", help="include the trained model")
+    p.add_argument("--pure_llm", action="store_true",
+                   help="include the pure-LLM baseline (series as text, no TS modality)")
     p.add_argument("--stage1_ckpt", default="checkpoints/stage1/stage1_final.pt")
     p.add_argument("--stage2_ckpt", default="checkpoints/stage2/stage2_final.pt")
     p.add_argument("--judge_backend", default="stub")
+    p.add_argument("--max_per_cat", type=int, default=0,
+                   help="cap samples per category for eval (0 = use all)")
     p.add_argument("--report_dir", default="reports")
     args = p.parse_args()
     run_eval(args)