fix(m5): eval pipeline — anomaly_regions schema + nested JSON + pure-LLM baseline

Critical eval-pipeline bugs found while running the stage-2 model: 1. parse_anomaly_segments only recognized {"segments\:[[s,e]]}; the training data (data/instruct.py) emits {"anomaly_regions":[{"start","end"}]}. The model learned the correct format but scored 0 → VUS-PR=0. Now accepts both schemas + a balanced-brace JSON scanner (the flat regex could not match nested objects the model emits). 2. qa_judge._extract_json had the same nesting bug — reuse the scanner. 3. Generation truncated at 64-96 tokens (too short for stats JSON) → bumped to 192 in report + instruct_check. Report enhancements: - wire the pure-LLM baseline (series-as-text, no TS modality) so design §5.4 'model vs pure LLM' is actually computed. Heavy predictors built/evaluated one at a time with GPU cleanup (single 12GB card). - add QA-judge table across all 6 categories (design §6.5 问答均分). - add --pure_llm / --max_per_cat flags. Results (reports/eval-2026-06-30.md): model QA-judge mean 2.45 vs pure_llm 0.88 (2.8x); anomaly localization (VUS-PR) is the weak point — neither model nor pure_llm localize well; TS-modality ablation change_rate 0.98.
2026-06-30 10:02:59 +00:00
parent 8aeddebce9
commit 40ec960cf8
8 changed files with 430 additions and 55 deletions
@@ -0,0 +1,33 @@
+# Eval Report — 2026-06-30
+
+Generated by `scripts/run_eval.sh` → `tsmm.eval.report`.
+
+## Discipline notes
+- **Main metric: VUS-PR** (threshold-free).
+- **Point-F1 reported WITHOUT point-adjust.** PA-F1 shown separately as a CONTROL only.
+- Trivial baselines (random / constant / all-report) are included.
+- LLM-judge backend: `stub`.
+
+## Dataset: synth
+
+| predictor | n | VUS-PR | Aff-F1 | AUC-PR | Point-F1 | PA-F1(control) | parse_rate |
+|---|---|---|---|---|---|---|---|
+| model | 40 | 0.1414 | 0.1674 | 0.2343 | 0.1414 | 0.4296 | 1.00 |
+| pure_llm | 40 | 0.2195 | 0.0000 | 0.2357 | 0.0000 | 0.0000 | 0.00 |
+| random | 40 | 0.2859 | 0.3484 | 0.2241 | 0.2859 | 0.9746 | 1.00 |
+| constant | 40 | 0.2195 | 0.3771 | 0.2357 | 0.3395 | 1.0000 | 1.00 |
+| all_report | 40 | 0.2195 | 0.3771 | 0.2357 | 0.3395 | 1.0000 | 1.00 |
+
+_PA = point-adjust; shown as control only, not headline._
+
+### QA judge (stub 0-5) — synth
+
+| predictor | anomaly | compare | describe | event | forecast | root_cause | mean |
+|---|---|---|---|---|---|---|---|
+| model | 2.60 | 2.64 | 1.65 | 4.08 | 0.12 | 3.60 | 2.45 |
+| pure_llm | 0.04 | 1.21 | 0.17 | 1.57 | 0.13 | 2.18 | 0.88 |
+| random | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 |
+| constant | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 |
+| all_report | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 |
+
+_QA judge = offline stub token-overlap heuristic; swap `--judge_backend openai` for GPT-4o scoring._
@@ -0,0 +1,130 @@
+{
+  "summary": {
+    "anomaly_json_parse_rate": 1.0,
+    "anomaly_json_pass": true,
+    "overall_usability": 0.9166666666666666,
+    "overall_usability_pass": true
+  },
+  "per_category": {
+    "anomaly": {
+      "n": 10,
+      "parse_rate": 1.0,
+      "nonempty_rate": 1.0,
+      "rule_acc": 0.02923274723648774,
+      "judge_mean": 2.4099999999999997,
+      "usability": 1.0,
+      "examples": [
+        {
+          "pred": "{\"anomaly_regions\": [{\"start\": 10, \"end\": 93}, {\"start\": 245, \"end\": 286}], \"n_regions\": 2}",
+          "ref": "{\"anomaly_regions\": [{\"start\": 100, \"end\": 145}, {\"start\": 146, \"end\": 200}, {\"start\": 371, \"end\": 373}, {\"start\": 382, \"end\": 445}], \"n_regions\": 4}",
+          "parsed": true
+        },
+        {
+          "pred": "{\"anomaly_regions\": [{\"start\": 10, \"end\": 29}, {\"start\": 43, \"end\": 56}], \"n_regions\": 2}",
+          "ref": "{\"anomaly_regions\": [{\"start\": 130, \"end\": 202}, {\"start\": 274, \"end\": 328}, {\"start\": 344, \"end\": 346}], \"n_regions\": 3}",
+          "parsed": true
+        }
+      ]
+    },
+    "describe": {
+      "n": 10,
+      "parse_rate": 1.0,
+      "nonempty_rate": 1.0,
+      "rule_acc": 0.0,
+      "judge_mean": 1.72,
+      "usability": 1.0,
+      "examples": [
+        {
+          "pred": "{\"stats\": [{\"name\": \"Active connections\", \"unit\": \"conn\", \"mean\": 43.0162578900111, \"std\": 10.0440401101011, \"min\": 25.0000091902004, \"max\": 61.204120208029}, {",
+          "ref": "{\"stats\": [{\"name\": \"Active connections\", \"unit\": \"conn\", \"mean\": 70.08136858780824, \"std\": 5.166230550292378, \"min\": 60.15172120024713, \"max\": 84.5416429780042",
+          "parsed": true
+        },
+        {
+          "pred": "{\"stats\": [{\"name\": \"Queue length\", \"unit\": \"tasks\", \"mean\": 42.0167398510011, \"std\": 5.04020002110111, \"min\": 31.0000000000001, \"max\": 55.0604040420208}, {\"sta",
+          "ref": "{\"stats\": [{\"name\": \"Queue length\", \"unit\": \"tasks\", \"mean\": 71.13851266805797, \"std\": 5.420230450566408, \"min\": 51.888456943983975, \"max\": 85.02043191625414}, ",
+          "parsed": true
+        }
+      ]
+    },
+    "forecast": {
+      "n": 10,
+      "parse_rate": 0.5,
+      "nonempty_rate": 0.5,
+      "rule_acc": 0.0,
+      "judge_mean": 0.14,
+      "usability": 0.5,
+      "examples": [
+        {
+          "pred": "{\"stats\": [{\"name\": \"Disk I/O\", \"unit\": \"MB/s\", \"mean\": 43.0125678911111, \"std\": 10.0404029110121, \"min\": 27.0000001200001, \"max\": 61.912222028021}, {\"name\": \"M",
+          "ref": "{\"forecast\": [[61.36301953352816, 61.372689003454326, 61.382358473380485, 61.392027943306644, 61.40169741323281, 61.41136688315897, 61.42103635308513, 61.430705",
+          "parsed": true
+        },
+        {
+          "pred": "{\"stats\": [{\"name\": \"Disk I/O\", \"unit\": \"MB/s\", \"mean\": 43.0125678912111, \"std\": 10.0400219204111, \"min\": 25.0000000200004, \"max\": 61.821123021212}, {\"name\": \"B",
+          "ref": "{\"forecast\": [[35.87848293955361, 35.69287951372365, 35.5072760878937, 35.321672662063754, 35.1360692362338, 34.95046581040385, 34.764862384573895, 34.579258958",
+          "parsed": true
+        }
+      ]
+    },
+    "root_cause": {
+      "n": 10,
+      "parse_rate": 1.0,
+      "nonempty_rate": 1.0,
+      "rule_acc": null,
+      "judge_mean": 3.75,
+      "usability": 1.0,
+      "examples": [
+        {
+          "pred": "The deviation in Active connections is most consistent with a sudden level shift. A plausible root cause is a deployment or configuration change that altered th",
+          "ref": "The deviation in Active connections is most consistent with an anomaly. A plausible root cause is a deployment or configuration change that altered the operatin",
+          "parsed": true
+        },
+        {
+          "pred": "The deviation in CPU usage is most consistent with a variance expansion. A plausible root cause is a deployment or configuration change that altered the operati",
+          "ref": "The deviation in CPU usage is most consistent with a gradual drift. A plausible root cause is a deployment or configuration change that altered the operating po",
+          "parsed": true
+        }
+      ]
+    },
+    "event": {
+      "n": 10,
+      "parse_rate": 1.0,
+      "nonempty_rate": 1.0,
+      "rule_acc": null,
+      "judge_mean": 4.5,
+      "usability": 1.0,
+      "examples": [
+        {
+          "pred": "No known event corresponds to the observed change.",
+          "ref": "No known event corresponds to the observed change.",
+          "parsed": true
+        },
+        {
+          "pred": "No known event corresponds to the observed change.",
+          "ref": "No known event corresponds to the observed change.",
+          "parsed": true
+        }
+      ]
+    },
+    "compare": {
+      "n": 10,
+      "parse_rate": 1.0,
+      "nonempty_rate": 1.0,
+      "rule_acc": null,
+      "judge_mean": 2.3899999999999997,
+      "usability": 1.0,
+      "examples": [
+        {
+          "pred": "Per-channel summary:\nActive connections: mean 30.26conn, range [15.47, 48.90];\nGPU utilization: mean 32.01%, range [19.00, 48.91];\nBattery level: mean 55.01%, r",
+          "ref": "Per-channel summary:\nActive connections: mean 57.19conn, range [28.39, 84.93]\nGPU utilization: mean 52.88%, range [17.81, 92.55]\nBattery level: mean 49.57%, ran",
+          "parsed": true
+        },
+        {
+          "pred": "Per-channel summary:\nCPU usage: mean 36.04%, range [12.57, 58.97]; Request latency p50: mean 32.04ms; Queue length: mean 31.05tasks; Disk I/O: mean 40.07MB/s; R",
+          "ref": "Per-channel summary:\nCPU usage: mean 33.10%, range [1.57, 54.50]\nRequest latency p50: mean 35.22ms, range [9.60, 59.28]\nQueue length: mean 26.72tasks, range [12",
+          "parsed": true
+        }
+      ]
+    }
+  }
+}
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# M4 + M5 exit-validation eval (T4.2 / T4.3 / T5.6).
+#
+# Runs (1) the instruction-following check (T4.2) and (2) the full eval report
+# (T5.5/5.6) including the trained model + trivial baselines, on the stage-2
+# checkpoint. Requires a free GPU (so run AFTER training finishes).
+#
+# Usage:
+#   scripts/run_m4_m5_eval.sh [stage2_ckpt]
+set -euo pipefail
+cd "$(dirname "$0")/.."
+export PYTHONPATH=src
+
+CKPT="${1:-checkpoints/stage2/stage2_final.pt}"
+if [ ! -f "$CKPT" ]; then
+  echo "[run_m4_m5_eval] checkpoint $CKPT not found"; exit 1
+fi
+
+echo "=== T4.2 instruction-following check (6 cats × 10) ==="
+.venv/bin/python -m tsmm.eval.instruct_check \
+  --data data/eval_synth.jsonl \
+  --per_cat 10 \
+  --stage2_ckpt "$CKPT" \
+  --judge_backend stub \
+  --out reports/instruct_check.json
+
+echo
+echo "=== T5.5/T5.6 full eval report (model + trivial baselines) ==="
+.venv/bin/python -m tsmm.eval.report \
+  --synth data/eval_synth.jsonl \
+  --model \
+  --stage2_ckpt "$CKPT" \
+  --judge_backend stub \
+  --report_dir reports
+
+echo
+echo "[run_m4_m5_eval] done. See reports/."
@@ -76,7 +76,7 @@ def _generate(m, col, sample) -> str:
    import torch
    batch = col([sample])
    with torch.no_grad():
-        txt = m.generate(batch, max_new_tokens=96)
+        txt = m.generate(batch, max_new_tokens=192)
    return txt[0] if txt else ""


@@ -115,12 +115,21 @@ def evaluate(samples_by_cat, model, col, judge_backend="stub") -> Dict[str, Dict
            if pred.strip():
                nonempty += 1

+            # describe expects a dict ref; parse JSON if possible, else {}
+            ref_for_describe = ref
+            if cat == "describe":
+                try:
+                    ref_for_describe = json.loads(ref) if isinstance(ref, str) else (ref or {})
+                    if not isinstance(ref_for_describe, dict):
+                        ref_for_describe = {}
+                except (ValueError, TypeError):
+                    ref_for_describe = {}
            r = rule_j.judge(
                cat, pred,
                ref_segs=[(seg.get("start", 0), seg.get("end", 0))
                          for seg in s.get("segments", [])],
                T=len(s.get("labels", [])) or 1,
-                ref_answer=ref if cat == "describe" else "",
+                ref_answer=ref_for_describe,
                ref_forecast=_forecast_list(ref) if cat == "forecast" else [],
            )
            if r.get("score") is not None:
@@ -155,6 +164,8 @@ def _forecast_list(ref_answer):
    """Extract the forecast value list from a forecast ref answer JSON."""
    try:
        obj = json.loads(ref_answer) if isinstance(ref_answer, str) else ref_answer
+        if not isinstance(obj, dict):
+            return []
        fc = obj.get("forecast")
        if isinstance(fc, list) and fc:
            inner = fc[0]
@@ -21,10 +21,52 @@ import numpy as np

 # match [int, int] or [[int, int]] style intervals
 _PAIR_RE = re.compile(r"\[\s*(\d+)\s*,\s*(\d+)\s*\]")
-# find JSON-ish objects in text
+# find JSON-ish objects in text (flat blobs only; nested handled by scanner)
 _JSON_RE = re.compile(r"\{[^{}]*\}", re.DOTALL)


+def _extract_balanced_json(text: str) -> List[str]:
+    """Yield every balanced ``{...}`` substring (handles nesting).
+
+    Scans the text for ``{`` and tracks depth, emitting the span for each
+    top-level balanced object. This lets us parse nested JSON the model emits
+    (e.g. ``{"anomaly_regions": [{"start": 10, "end": 93}]}``) which the
+    flat ``_JSON_RE`` cannot match.
+    """
+    out = []
+    depth = 0
+    start = -1
+    instr = False
+    esc = False
+    quote = ""
+    for i, ch in enumerate(text):
+        if esc:
+            esc = False
+            continue
+        if ch == "\\":
+            esc = True
+            continue
+        if instr:
+            if ch == quote:
+                instr = False
+            continue
+        if ch in ("'", '"'):
+            instr = True
+            quote = ch
+            continue
+        if ch == "{":
+            if depth == 0:
+                start = i
+            depth += 1
+        elif ch == "}":
+            if depth > 0:
+                depth -= 1
+                if depth == 0 and start >= 0:
+                    out.append(text[start : i + 1])
+                    start = -1
+    return out
+
+
 def _normalize_seg(pair) -> Tuple[int, int]:
    a, b = int(pair[0]), int(pair[1])
    if a > b:
@@ -32,36 +74,59 @@ def _normalize_seg(pair) -> Tuple[int, int]:
    return (a, b)


+def _seg_from_entry(s):
+    """Extract a (start, end) tuple from one segment entry.
+
+    Accepts either ``[start, end]`` lists/tuples or ``{"start": s, "end": e}``
+    objects (the schema emitted by ``data/instruct.py``). Returns None if no
+    usable pair is found.
+    """
+    if isinstance(s, dict):
+        if "start" in s and "end" in s:
+            return (int(s["start"]), int(s["end"]))
+        return None
+    if isinstance(s, (list, tuple)):
+        if len(s) >= 2:
+            return _normalize_seg(s)
+        if len(s) == 1:
+            v = int(s[0]); return (v, v)
+    return None
+
+
 def parse_anomaly_segments(text: str) -> Tuple[List[Tuple[int, int]], bool]:
-    """Return (segments, json_ok). json_ok=True iff a JSON ``segments`` key found."""
+    """Return (segments, json_ok).
+
+    json_ok=True iff a JSON object with a recognized segments key was found.
+    Recognized keys: ``segments`` (list of [s,e]) and ``anomaly_regions``
+    (list of {"start":s,"end":e} — the schema used by ``data/instruct.py``).
+    """
    if text is None:
        return [], False
-    # 1. try every {...} blob; prefer one with a 'segments' key
-    candidates = _JSON_RE.findall(text)
+    # 1. try every balanced {...} blob; prefer one with a recognized segments key
+    candidates = _extract_balanced_json(text)
    best_segments = None
    for blob in candidates:
        try:
            obj = json.loads(blob)
        except json.JSONDecodeError:
            continue
-        if isinstance(obj, dict) and "segments" in obj:
-            segs = obj["segments"]
-            if not isinstance(segs, list):
+        if not isinstance(obj, dict):
            continue
-            out = []
-            for s in segs:
-                if isinstance(s, (list, tuple)):
-                    if len(s) >= 2:
-                        out.append(_normalize_seg(s))
-                    elif len(s) == 1:
-                        v = int(s[0]); out.append((v, v))
-                # ignore non-list entries
+        raw = None
+        if "segments" in obj:
+            raw = obj["segments"]
+        elif "anomaly_regions" in obj:
+            raw = obj["anomaly_regions"]
+        if raw is None or not isinstance(raw, list):
+            continue
+        out = [_seg_from_entry(s) for s in raw]
+        out = [seg for seg in out if seg is not None]
        best_segments = out
        break
    if best_segments is not None:
        return best_segments, True

-    # 2. fallback: regex-extract [int,int] pairs (no valid JSON segments found)
+    # 2. fallback: regex-extract [int,int] pairs (no valid JSON key found)
    pairs = _PAIR_RE.findall(text)
    if pairs:
        return [_normalize_seg(p) for p in pairs], False
@@ -16,7 +16,7 @@ import json
 import re
 from typing import Any, Callable, Dict, List, Optional, Tuple

-from .parse_answer import parse_anomaly_segments
+from .parse_answer import _extract_balanced_json, parse_anomaly_segments


 _JSON_RE = re.compile(r"\{[^{}]*\}", re.DOTALL)
@@ -25,6 +25,15 @@ _JSON_RE = re.compile(r"\{[^{}]*\}", re.DOTALL)
 def _extract_json(pred: str) -> Optional[dict]:
    if pred is None:
        return None
+    # prefer balanced (nested-capable) extraction first
+    for blob in _extract_balanced_json(pred):
+        try:
+            obj = json.loads(blob)
+            if isinstance(obj, dict):
+                return obj
+        except json.JSONDecodeError:
+            continue
+    # fallback to flat blobs
    for blob in _JSON_RE.findall(pred):
        try:
            return json.loads(blob)
@@ -157,14 +157,18 @@ def run_eval(args: argparse.Namespace) -> str:
    if args.real and os.path.exists(args.real):
        datasets.append(("real", args.real))

-    # predictors: trained model (optional) + trivial baselines
+    # predictors: trained model (optional) + trivial baselines + pure-LLM
    predictors: Dict[str, Any] = {
        "random": trivial_predictor(RandomBaseline(seed=0)),
        "constant": trivial_predictor(ConstantBaseline(0.5)),
        "all_report": trivial_predictor(AllReportBaseline()),
    }
+
+    # Heavy predictors are built & evaluated one at a time with GPU cleanup so
+    # we never hold two full models on a single 12GB card.
+    heavy_builders: Dict[str, Any] = {}  # name -> zero-arg callable returning predictor fn
    if args.model:
-        # lazy heavy import
+        def _build_trained():
            import torch
            from ..model.ts_encoder import TSEncoder
            from ..model.projector import Projector
@@ -188,32 +192,86 @@ def run_eval(args: argparse.Namespace) -> str:
            m.eval()
            col = Collator(max_T=512)

-        def _model_pred(sample, _m=m, _col=col):
+            def _pred(sample, _m=m, _col=col):
                batch = _col([sample])
                with torch.no_grad():
-                txt = _m.generate(batch, max_new_tokens=64)
+                    txt = _m.generate(batch, max_new_tokens=192)
                return txt[0] if txt else ""
+            return _pred

-        predictors = {"model": _model_pred, **predictors}
+        def _build_pure_llm():
+            # base LLM, series fed as TEXT (no TS modality, no LoRA) — design §5.4
+            import torch
+            from ..model.ts_encoder import TSEncoder
+            from ..model.projector import Projector
+            from ..model.wrapper import MultimodalTSModel
+            LLM = os.environ.get("TSMM_LLM_PATH", "/home/zhangzp/models/Qwen2.5-0.5B-Instruct")
+            m = MultimodalTSModel(LLM, TSEncoder(), Projector(), dtype=torch.bfloat16).cuda()
+            m.eval()
+            tok = m.tokenizer
+
+            def _pred(sample, _m=m, _tok=tok):
+                from .baselines import pure_llm_answer
+                return pure_llm_answer(
+                    sample.get("series", []),
+                    sample.get("instruction") or sample.get("question") or "",
+                    model=_m, max_new_tokens=192)
+            return _pred
+
+        heavy_builders["model"] = _build_trained
+        if args.pure_llm:
+            heavy_builders["pure_llm"] = _build_pure_llm

    # run anomaly eval per dataset × predictor
    results: Dict[str, Dict[str, Any]] = defaultdict(dict)
    qa_results: Dict[str, Dict[str, Dict[str, Any]]] = defaultdict(lambda: defaultdict(dict))
+    # First pass: cache per-dataset splits so heavy predictors reuse them.
+    splits: Dict[str, Dict[str, List[dict]]] = {}
+    maxpc = args.max_per_cat
    for ds_name, path in datasets:
        samples = list(stream_jsonl(path))
        by_cat: Dict[str, List[dict]] = defaultdict(list)
        for s in samples:
            by_cat[s.get("category", "?")].append(s)
+        if maxpc and maxpc > 0:
+            for cat in list(by_cat.keys()):
+                by_cat[cat] = by_cat[cat][:maxpc]
+        splits[ds_name] = by_cat
+
+    def _eval_predictor(pname, pred):
+        for ds_name, by_cat in splits.items():
            anom = by_cat.get("anomaly", [])
-        for pname, pred in predictors.items():
            results[ds_name][pname] = eval_anomaly_task(anom, pred)
-            # QA judge across all categories (covers design §6.5 问答均分)
            for cat, cat_samples in by_cat.items():
                qa_results[ds_name][pname][cat] = eval_qa_task(
                    cat_samples, pred, judge_backend=args.judge_backend)

+    # trivial (CPU) predictors first
+    all_pred_names = list(predictors.keys())
+    for pname, pred in predictors.items():
+        _eval_predictor(pname, pred)
+
+    # heavy predictors one at a time, with GPU cleanup between
+    if heavy_builders:
+        import torch
+        import gc
+        for pname, builder in heavy_builders.items():
+            print(f"[run_eval] building heavy predictor: {pname}")
+            pred = builder()
+            _eval_predictor(pname, pred)
+            all_pred_names.append(pname)
+            # free GPU before the next heavy model
+            del pred
+            gc.collect()
+            torch.cuda.empty_cache()
+
+    predictor_order = [n for n in ["model", "pure_llm"] if n in all_pred_names]
+    for n in all_pred_names:
+        if n not in predictor_order:
+            predictor_order.append(n)
+
    # render markdown
-    return _render_report(results, qa_results, datasets, predictors.keys(), args)
+    return _render_report(results, qa_results, datasets, predictor_order, args)


 def _render_report(results, qa_results, datasets, predictor_names, args) -> str:
@@ -293,9 +351,13 @@ def main() -> None:
    p.add_argument("--synth", default="data/eval_synth.jsonl")
    p.add_argument("--real", default="data/eval_real.jsonl")
    p.add_argument("--model", action="store_true", help="include the trained model")
+    p.add_argument("--pure_llm", action="store_true",
+                   help="include the pure-LLM baseline (series as text, no TS modality)")
    p.add_argument("--stage1_ckpt", default="checkpoints/stage1/stage1_final.pt")
    p.add_argument("--stage2_ckpt", default="checkpoints/stage2/stage2_final.pt")
    p.add_argument("--judge_backend", default="stub")
+    p.add_argument("--max_per_cat", type=int, default=0,
+                   help="cap samples per category for eval (0 = use all)")
    p.add_argument("--report_dir", default="reports")
    args = p.parse_args()
    run_eval(args)
@@ -45,6 +45,34 @@ class TestParseSegments:
        # ok=False because no valid JSON, but segments extracted via fallback
        assert segs == [(100, 150), (200, 250)]

+    def test_anomaly_regions_schema(self):
+        # the schema data/instruct.py actually emits (nested, start/end objects)
+        text = '{"anomaly_regions": [{"start": 10, "end": 93}, {"start": 245, "end": 286}], "n_regions": 2}'
+        segs, ok = parse_anomaly_segments(text)
+        assert ok is True
+        assert segs == [(10, 93), (245, 286)]
+
+    def test_anomaly_regions_in_prose(self):
+        text = 'Based on the series: {"anomaly_regions": [{"start": 5, "end": 8}]} done.'
+        segs, ok = parse_anomaly_segments(text)
+        assert ok is True
+        assert segs == [(5, 8)]
+
+    def test_segments_with_start_end_objects(self):
+        # also accept the {start,end} object form under the 'segments' key
+        text = '{"segments": [{"start": 1, "end": 4}]}'
+        segs, ok = parse_anomaly_segments(text)
+        assert ok is True
+        assert segs == [(1, 4)]
+
+    def test_balanced_json_handles_nested(self):
+        from tsmm.eval.parse_answer import _extract_balanced_json
+        text = 'x {"a": {"b": 1}, "c": [2,3]} y {"d": 4}'
+        blobs = _extract_balanced_json(text)
+        # outer nested object + the flat one, but NOT the inner {"b":1} alone
+        assert any('"a"' in b and '"b"' in b for b in blobs)
+        assert '{"d": 4}' in blobs
+
    def test_empty_segments_list(self):
        text = '{"segments": [], "type": "none"}'
        segs, ok = parse_anomaly_segments(text)