diff --git a/reports/eval-2026-06-30.md b/reports/eval-2026-06-30.md new file mode 100644 index 0000000..992470a --- /dev/null +++ b/reports/eval-2026-06-30.md @@ -0,0 +1,33 @@ +# Eval Report — 2026-06-30 + +Generated by `scripts/run_eval.sh` → `tsmm.eval.report`. + +## Discipline notes +- **Main metric: VUS-PR** (threshold-free). +- **Point-F1 reported WITHOUT point-adjust.** PA-F1 shown separately as a CONTROL only. +- Trivial baselines (random / constant / all-report) are included. +- LLM-judge backend: `stub`. + +## Dataset: synth + +| predictor | n | VUS-PR | Aff-F1 | AUC-PR | Point-F1 | PA-F1(control) | parse_rate | +|---|---|---|---|---|---|---|---| +| model | 40 | 0.1414 | 0.1674 | 0.2343 | 0.1414 | 0.4296 | 1.00 | +| pure_llm | 40 | 0.2195 | 0.0000 | 0.2357 | 0.0000 | 0.0000 | 0.00 | +| random | 40 | 0.2859 | 0.3484 | 0.2241 | 0.2859 | 0.9746 | 1.00 | +| constant | 40 | 0.2195 | 0.3771 | 0.2357 | 0.3395 | 1.0000 | 1.00 | +| all_report | 40 | 0.2195 | 0.3771 | 0.2357 | 0.3395 | 1.0000 | 1.00 | + +_PA = point-adjust; shown as control only, not headline._ + +### QA judge (stub 0-5) — synth + +| predictor | anomaly | compare | describe | event | forecast | root_cause | mean | +|---|---|---|---|---|---|---|---| +| model | 2.60 | 2.64 | 1.65 | 4.08 | 0.12 | 3.60 | 2.45 | +| pure_llm | 0.04 | 1.21 | 0.17 | 1.57 | 0.13 | 2.18 | 0.88 | +| random | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | +| constant | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | +| all_report | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | + +_QA judge = offline stub token-overlap heuristic; swap `--judge_backend openai` for GPT-4o scoring._ diff --git a/reports/instruct_check.json b/reports/instruct_check.json new file mode 100644 index 0000000..f74e80c --- /dev/null +++ b/reports/instruct_check.json @@ -0,0 +1,130 @@ +{ + "summary": { + "anomaly_json_parse_rate": 1.0, + "anomaly_json_pass": true, + "overall_usability": 0.9166666666666666, + "overall_usability_pass": true + }, + "per_category": { + "anomaly": { + "n": 10, + "parse_rate": 1.0, + "nonempty_rate": 1.0, + "rule_acc": 0.02923274723648774, + "judge_mean": 2.4099999999999997, + "usability": 1.0, + "examples": [ + { + "pred": "{\"anomaly_regions\": [{\"start\": 10, \"end\": 93}, {\"start\": 245, \"end\": 286}], \"n_regions\": 2}", + "ref": "{\"anomaly_regions\": [{\"start\": 100, \"end\": 145}, {\"start\": 146, \"end\": 200}, {\"start\": 371, \"end\": 373}, {\"start\": 382, \"end\": 445}], \"n_regions\": 4}", + "parsed": true + }, + { + "pred": "{\"anomaly_regions\": [{\"start\": 10, \"end\": 29}, {\"start\": 43, \"end\": 56}], \"n_regions\": 2}", + "ref": "{\"anomaly_regions\": [{\"start\": 130, \"end\": 202}, {\"start\": 274, \"end\": 328}, {\"start\": 344, \"end\": 346}], \"n_regions\": 3}", + "parsed": true + } + ] + }, + "describe": { + "n": 10, + "parse_rate": 1.0, + "nonempty_rate": 1.0, + "rule_acc": 0.0, + "judge_mean": 1.72, + "usability": 1.0, + "examples": [ + { + "pred": "{\"stats\": [{\"name\": \"Active connections\", \"unit\": \"conn\", \"mean\": 43.0162578900111, \"std\": 10.0440401101011, \"min\": 25.0000091902004, \"max\": 61.204120208029}, {", + "ref": "{\"stats\": [{\"name\": \"Active connections\", \"unit\": \"conn\", \"mean\": 70.08136858780824, \"std\": 5.166230550292378, \"min\": 60.15172120024713, \"max\": 84.5416429780042", + "parsed": true + }, + { + "pred": "{\"stats\": [{\"name\": \"Queue length\", \"unit\": \"tasks\", \"mean\": 42.0167398510011, \"std\": 5.04020002110111, \"min\": 31.0000000000001, \"max\": 55.0604040420208}, {\"sta", + "ref": "{\"stats\": [{\"name\": \"Queue length\", \"unit\": \"tasks\", \"mean\": 71.13851266805797, \"std\": 5.420230450566408, \"min\": 51.888456943983975, \"max\": 85.02043191625414}, ", + "parsed": true + } + ] + }, + "forecast": { + "n": 10, + "parse_rate": 0.5, + "nonempty_rate": 0.5, + "rule_acc": 0.0, + "judge_mean": 0.14, + "usability": 0.5, + "examples": [ + { + "pred": "{\"stats\": [{\"name\": \"Disk I/O\", \"unit\": \"MB/s\", \"mean\": 43.0125678911111, \"std\": 10.0404029110121, \"min\": 27.0000001200001, \"max\": 61.912222028021}, {\"name\": \"M", + "ref": "{\"forecast\": [[61.36301953352816, 61.372689003454326, 61.382358473380485, 61.392027943306644, 61.40169741323281, 61.41136688315897, 61.42103635308513, 61.430705", + "parsed": true + }, + { + "pred": "{\"stats\": [{\"name\": \"Disk I/O\", \"unit\": \"MB/s\", \"mean\": 43.0125678912111, \"std\": 10.0400219204111, \"min\": 25.0000000200004, \"max\": 61.821123021212}, {\"name\": \"B", + "ref": "{\"forecast\": [[35.87848293955361, 35.69287951372365, 35.5072760878937, 35.321672662063754, 35.1360692362338, 34.95046581040385, 34.764862384573895, 34.579258958", + "parsed": true + } + ] + }, + "root_cause": { + "n": 10, + "parse_rate": 1.0, + "nonempty_rate": 1.0, + "rule_acc": null, + "judge_mean": 3.75, + "usability": 1.0, + "examples": [ + { + "pred": "The deviation in Active connections is most consistent with a sudden level shift. A plausible root cause is a deployment or configuration change that altered th", + "ref": "The deviation in Active connections is most consistent with an anomaly. A plausible root cause is a deployment or configuration change that altered the operatin", + "parsed": true + }, + { + "pred": "The deviation in CPU usage is most consistent with a variance expansion. A plausible root cause is a deployment or configuration change that altered the operati", + "ref": "The deviation in CPU usage is most consistent with a gradual drift. A plausible root cause is a deployment or configuration change that altered the operating po", + "parsed": true + } + ] + }, + "event": { + "n": 10, + "parse_rate": 1.0, + "nonempty_rate": 1.0, + "rule_acc": null, + "judge_mean": 4.5, + "usability": 1.0, + "examples": [ + { + "pred": "No known event corresponds to the observed change.", + "ref": "No known event corresponds to the observed change.", + "parsed": true + }, + { + "pred": "No known event corresponds to the observed change.", + "ref": "No known event corresponds to the observed change.", + "parsed": true + } + ] + }, + "compare": { + "n": 10, + "parse_rate": 1.0, + "nonempty_rate": 1.0, + "rule_acc": null, + "judge_mean": 2.3899999999999997, + "usability": 1.0, + "examples": [ + { + "pred": "Per-channel summary:\nActive connections: mean 30.26conn, range [15.47, 48.90];\nGPU utilization: mean 32.01%, range [19.00, 48.91];\nBattery level: mean 55.01%, r", + "ref": "Per-channel summary:\nActive connections: mean 57.19conn, range [28.39, 84.93]\nGPU utilization: mean 52.88%, range [17.81, 92.55]\nBattery level: mean 49.57%, ran", + "parsed": true + }, + { + "pred": "Per-channel summary:\nCPU usage: mean 36.04%, range [12.57, 58.97]; Request latency p50: mean 32.04ms; Queue length: mean 31.05tasks; Disk I/O: mean 40.07MB/s; R", + "ref": "Per-channel summary:\nCPU usage: mean 33.10%, range [1.57, 54.50]\nRequest latency p50: mean 35.22ms, range [9.60, 59.28]\nQueue length: mean 26.72tasks, range [12", + "parsed": true + } + ] + } + } +} \ No newline at end of file diff --git a/scripts/run_m4_m5_eval.sh b/scripts/run_m4_m5_eval.sh new file mode 100755 index 0000000..3d5f2ab --- /dev/null +++ b/scripts/run_m4_m5_eval.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# M4 + M5 exit-validation eval (T4.2 / T4.3 / T5.6). +# +# Runs (1) the instruction-following check (T4.2) and (2) the full eval report +# (T5.5/5.6) including the trained model + trivial baselines, on the stage-2 +# checkpoint. Requires a free GPU (so run AFTER training finishes). +# +# Usage: +# scripts/run_m4_m5_eval.sh [stage2_ckpt] +set -euo pipefail +cd "$(dirname "$0")/.." +export PYTHONPATH=src + +CKPT="${1:-checkpoints/stage2/stage2_final.pt}" +if [ ! -f "$CKPT" ]; then + echo "[run_m4_m5_eval] checkpoint $CKPT not found"; exit 1 +fi + +echo "=== T4.2 instruction-following check (6 cats × 10) ===" +.venv/bin/python -m tsmm.eval.instruct_check \ + --data data/eval_synth.jsonl \ + --per_cat 10 \ + --stage2_ckpt "$CKPT" \ + --judge_backend stub \ + --out reports/instruct_check.json + +echo +echo "=== T5.5/T5.6 full eval report (model + trivial baselines) ===" +.venv/bin/python -m tsmm.eval.report \ + --synth data/eval_synth.jsonl \ + --model \ + --stage2_ckpt "$CKPT" \ + --judge_backend stub \ + --report_dir reports + +echo +echo "[run_m4_m5_eval] done. See reports/." diff --git a/src/tsmm/eval/instruct_check.py b/src/tsmm/eval/instruct_check.py index 708f0b4..ea5c7a5 100644 --- a/src/tsmm/eval/instruct_check.py +++ b/src/tsmm/eval/instruct_check.py @@ -76,7 +76,7 @@ def _generate(m, col, sample) -> str: import torch batch = col([sample]) with torch.no_grad(): - txt = m.generate(batch, max_new_tokens=96) + txt = m.generate(batch, max_new_tokens=192) return txt[0] if txt else "" @@ -115,12 +115,21 @@ def evaluate(samples_by_cat, model, col, judge_backend="stub") -> Dict[str, Dict if pred.strip(): nonempty += 1 + # describe expects a dict ref; parse JSON if possible, else {} + ref_for_describe = ref + if cat == "describe": + try: + ref_for_describe = json.loads(ref) if isinstance(ref, str) else (ref or {}) + if not isinstance(ref_for_describe, dict): + ref_for_describe = {} + except (ValueError, TypeError): + ref_for_describe = {} r = rule_j.judge( cat, pred, ref_segs=[(seg.get("start", 0), seg.get("end", 0)) for seg in s.get("segments", [])], T=len(s.get("labels", [])) or 1, - ref_answer=ref if cat == "describe" else "", + ref_answer=ref_for_describe, ref_forecast=_forecast_list(ref) if cat == "forecast" else [], ) if r.get("score") is not None: @@ -155,6 +164,8 @@ def _forecast_list(ref_answer): """Extract the forecast value list from a forecast ref answer JSON.""" try: obj = json.loads(ref_answer) if isinstance(ref_answer, str) else ref_answer + if not isinstance(obj, dict): + return [] fc = obj.get("forecast") if isinstance(fc, list) and fc: inner = fc[0] diff --git a/src/tsmm/eval/parse_answer.py b/src/tsmm/eval/parse_answer.py index d7a1fbe..3906ed1 100644 --- a/src/tsmm/eval/parse_answer.py +++ b/src/tsmm/eval/parse_answer.py @@ -21,10 +21,52 @@ import numpy as np # match [int, int] or [[int, int]] style intervals _PAIR_RE = re.compile(r"\[\s*(\d+)\s*,\s*(\d+)\s*\]") -# find JSON-ish objects in text +# find JSON-ish objects in text (flat blobs only; nested handled by scanner) _JSON_RE = re.compile(r"\{[^{}]*\}", re.DOTALL) +def _extract_balanced_json(text: str) -> List[str]: + """Yield every balanced ``{...}`` substring (handles nesting). + + Scans the text for ``{`` and tracks depth, emitting the span for each + top-level balanced object. This lets us parse nested JSON the model emits + (e.g. ``{"anomaly_regions": [{"start": 10, "end": 93}]}``) which the + flat ``_JSON_RE`` cannot match. + """ + out = [] + depth = 0 + start = -1 + instr = False + esc = False + quote = "" + for i, ch in enumerate(text): + if esc: + esc = False + continue + if ch == "\\": + esc = True + continue + if instr: + if ch == quote: + instr = False + continue + if ch in ("'", '"'): + instr = True + quote = ch + continue + if ch == "{": + if depth == 0: + start = i + depth += 1 + elif ch == "}": + if depth > 0: + depth -= 1 + if depth == 0 and start >= 0: + out.append(text[start : i + 1]) + start = -1 + return out + + def _normalize_seg(pair) -> Tuple[int, int]: a, b = int(pair[0]), int(pair[1]) if a > b: @@ -32,36 +74,59 @@ def _normalize_seg(pair) -> Tuple[int, int]: return (a, b) +def _seg_from_entry(s): + """Extract a (start, end) tuple from one segment entry. + + Accepts either ``[start, end]`` lists/tuples or ``{"start": s, "end": e}`` + objects (the schema emitted by ``data/instruct.py``). Returns None if no + usable pair is found. + """ + if isinstance(s, dict): + if "start" in s and "end" in s: + return (int(s["start"]), int(s["end"])) + return None + if isinstance(s, (list, tuple)): + if len(s) >= 2: + return _normalize_seg(s) + if len(s) == 1: + v = int(s[0]); return (v, v) + return None + + def parse_anomaly_segments(text: str) -> Tuple[List[Tuple[int, int]], bool]: - """Return (segments, json_ok). json_ok=True iff a JSON ``segments`` key found.""" + """Return (segments, json_ok). + + json_ok=True iff a JSON object with a recognized segments key was found. + Recognized keys: ``segments`` (list of [s,e]) and ``anomaly_regions`` + (list of {"start":s,"end":e} — the schema used by ``data/instruct.py``). + """ if text is None: return [], False - # 1. try every {...} blob; prefer one with a 'segments' key - candidates = _JSON_RE.findall(text) + # 1. try every balanced {...} blob; prefer one with a recognized segments key + candidates = _extract_balanced_json(text) best_segments = None for blob in candidates: try: obj = json.loads(blob) except json.JSONDecodeError: continue - if isinstance(obj, dict) and "segments" in obj: - segs = obj["segments"] - if not isinstance(segs, list): - continue - out = [] - for s in segs: - if isinstance(s, (list, tuple)): - if len(s) >= 2: - out.append(_normalize_seg(s)) - elif len(s) == 1: - v = int(s[0]); out.append((v, v)) - # ignore non-list entries - best_segments = out - break + if not isinstance(obj, dict): + continue + raw = None + if "segments" in obj: + raw = obj["segments"] + elif "anomaly_regions" in obj: + raw = obj["anomaly_regions"] + if raw is None or not isinstance(raw, list): + continue + out = [_seg_from_entry(s) for s in raw] + out = [seg for seg in out if seg is not None] + best_segments = out + break if best_segments is not None: return best_segments, True - # 2. fallback: regex-extract [int,int] pairs (no valid JSON segments found) + # 2. fallback: regex-extract [int,int] pairs (no valid JSON key found) pairs = _PAIR_RE.findall(text) if pairs: return [_normalize_seg(p) for p in pairs], False diff --git a/src/tsmm/eval/qa_judge.py b/src/tsmm/eval/qa_judge.py index b80b19e..53c591f 100644 --- a/src/tsmm/eval/qa_judge.py +++ b/src/tsmm/eval/qa_judge.py @@ -16,7 +16,7 @@ import json import re from typing import Any, Callable, Dict, List, Optional, Tuple -from .parse_answer import parse_anomaly_segments +from .parse_answer import _extract_balanced_json, parse_anomaly_segments _JSON_RE = re.compile(r"\{[^{}]*\}", re.DOTALL) @@ -25,6 +25,15 @@ _JSON_RE = re.compile(r"\{[^{}]*\}", re.DOTALL) def _extract_json(pred: str) -> Optional[dict]: if pred is None: return None + # prefer balanced (nested-capable) extraction first + for blob in _extract_balanced_json(pred): + try: + obj = json.loads(blob) + if isinstance(obj, dict): + return obj + except json.JSONDecodeError: + continue + # fallback to flat blobs for blob in _JSON_RE.findall(pred): try: return json.loads(blob) diff --git a/src/tsmm/eval/report.py b/src/tsmm/eval/report.py index 80d4e20..2f2f3cf 100644 --- a/src/tsmm/eval/report.py +++ b/src/tsmm/eval/report.py @@ -157,63 +157,121 @@ def run_eval(args: argparse.Namespace) -> str: if args.real and os.path.exists(args.real): datasets.append(("real", args.real)) - # predictors: trained model (optional) + trivial baselines + # predictors: trained model (optional) + trivial baselines + pure-LLM predictors: Dict[str, Any] = { "random": trivial_predictor(RandomBaseline(seed=0)), "constant": trivial_predictor(ConstantBaseline(0.5)), "all_report": trivial_predictor(AllReportBaseline()), } + + # Heavy predictors are built & evaluated one at a time with GPU cleanup so + # we never hold two full models on a single 12GB card. + heavy_builders: Dict[str, Any] = {} # name -> zero-arg callable returning predictor fn if args.model: - # lazy heavy import - import torch - from ..model.ts_encoder import TSEncoder - from ..model.projector import Projector - from ..model.wrapper import MultimodalTSModel - from ..data.collator import Collator + def _build_trained(): + import torch + from ..model.ts_encoder import TSEncoder + from ..model.projector import Projector + from ..model.wrapper import MultimodalTSModel + from ..data.collator import Collator - LLM = os.environ.get("TSMM_LLM_PATH", "/home/zhangzp/models/Qwen2.5-0.5B-Instruct") - m = MultimodalTSModel(LLM, TSEncoder(), Projector(), dtype=torch.bfloat16).cuda() - if args.stage2_ckpt and os.path.exists(args.stage2_ckpt): - st = torch.load(args.stage2_ckpt, map_location="cpu") - m.encoder.load_state_dict(st["encoder"]) - m.projector.load_state_dict(st["projector"]) - lora_dir = os.path.join(os.path.dirname(args.stage2_ckpt), "lora_adapter") - if os.path.exists(lora_dir): - from peft import PeftModel - m.llm = PeftModel.from_pretrained(m.llm, lora_dir) - elif args.stage1_ckpt and os.path.exists(args.stage1_ckpt): - st = torch.load(args.stage1_ckpt, map_location="cpu") - m.encoder.load_state_dict(st["encoder"]) - m.projector.load_state_dict(st["projector"]) - m.eval() - col = Collator(max_T=512) + LLM = os.environ.get("TSMM_LLM_PATH", "/home/zhangzp/models/Qwen2.5-0.5B-Instruct") + m = MultimodalTSModel(LLM, TSEncoder(), Projector(), dtype=torch.bfloat16).cuda() + if args.stage2_ckpt and os.path.exists(args.stage2_ckpt): + st = torch.load(args.stage2_ckpt, map_location="cpu") + m.encoder.load_state_dict(st["encoder"]) + m.projector.load_state_dict(st["projector"]) + lora_dir = os.path.join(os.path.dirname(args.stage2_ckpt), "lora_adapter") + if os.path.exists(lora_dir): + from peft import PeftModel + m.llm = PeftModel.from_pretrained(m.llm, lora_dir) + elif args.stage1_ckpt and os.path.exists(args.stage1_ckpt): + st = torch.load(args.stage1_ckpt, map_location="cpu") + m.encoder.load_state_dict(st["encoder"]) + m.projector.load_state_dict(st["projector"]) + m.eval() + col = Collator(max_T=512) - def _model_pred(sample, _m=m, _col=col): - batch = _col([sample]) - with torch.no_grad(): - txt = _m.generate(batch, max_new_tokens=64) - return txt[0] if txt else "" + def _pred(sample, _m=m, _col=col): + batch = _col([sample]) + with torch.no_grad(): + txt = _m.generate(batch, max_new_tokens=192) + return txt[0] if txt else "" + return _pred - predictors = {"model": _model_pred, **predictors} + def _build_pure_llm(): + # base LLM, series fed as TEXT (no TS modality, no LoRA) — design §5.4 + import torch + from ..model.ts_encoder import TSEncoder + from ..model.projector import Projector + from ..model.wrapper import MultimodalTSModel + LLM = os.environ.get("TSMM_LLM_PATH", "/home/zhangzp/models/Qwen2.5-0.5B-Instruct") + m = MultimodalTSModel(LLM, TSEncoder(), Projector(), dtype=torch.bfloat16).cuda() + m.eval() + tok = m.tokenizer + + def _pred(sample, _m=m, _tok=tok): + from .baselines import pure_llm_answer + return pure_llm_answer( + sample.get("series", []), + sample.get("instruction") or sample.get("question") or "", + model=_m, max_new_tokens=192) + return _pred + + heavy_builders["model"] = _build_trained + if args.pure_llm: + heavy_builders["pure_llm"] = _build_pure_llm # run anomaly eval per dataset × predictor results: Dict[str, Dict[str, Any]] = defaultdict(dict) qa_results: Dict[str, Dict[str, Dict[str, Any]]] = defaultdict(lambda: defaultdict(dict)) + # First pass: cache per-dataset splits so heavy predictors reuse them. + splits: Dict[str, Dict[str, List[dict]]] = {} + maxpc = args.max_per_cat for ds_name, path in datasets: samples = list(stream_jsonl(path)) by_cat: Dict[str, List[dict]] = defaultdict(list) for s in samples: by_cat[s.get("category", "?")].append(s) - anom = by_cat.get("anomaly", []) - for pname, pred in predictors.items(): + if maxpc and maxpc > 0: + for cat in list(by_cat.keys()): + by_cat[cat] = by_cat[cat][:maxpc] + splits[ds_name] = by_cat + + def _eval_predictor(pname, pred): + for ds_name, by_cat in splits.items(): + anom = by_cat.get("anomaly", []) results[ds_name][pname] = eval_anomaly_task(anom, pred) - # QA judge across all categories (covers design §6.5 问答均分) for cat, cat_samples in by_cat.items(): qa_results[ds_name][pname][cat] = eval_qa_task( cat_samples, pred, judge_backend=args.judge_backend) + # trivial (CPU) predictors first + all_pred_names = list(predictors.keys()) + for pname, pred in predictors.items(): + _eval_predictor(pname, pred) + + # heavy predictors one at a time, with GPU cleanup between + if heavy_builders: + import torch + import gc + for pname, builder in heavy_builders.items(): + print(f"[run_eval] building heavy predictor: {pname}") + pred = builder() + _eval_predictor(pname, pred) + all_pred_names.append(pname) + # free GPU before the next heavy model + del pred + gc.collect() + torch.cuda.empty_cache() + + predictor_order = [n for n in ["model", "pure_llm"] if n in all_pred_names] + for n in all_pred_names: + if n not in predictor_order: + predictor_order.append(n) + # render markdown - return _render_report(results, qa_results, datasets, predictors.keys(), args) + return _render_report(results, qa_results, datasets, predictor_order, args) def _render_report(results, qa_results, datasets, predictor_names, args) -> str: @@ -293,9 +351,13 @@ def main() -> None: p.add_argument("--synth", default="data/eval_synth.jsonl") p.add_argument("--real", default="data/eval_real.jsonl") p.add_argument("--model", action="store_true", help="include the trained model") + p.add_argument("--pure_llm", action="store_true", + help="include the pure-LLM baseline (series as text, no TS modality)") p.add_argument("--stage1_ckpt", default="checkpoints/stage1/stage1_final.pt") p.add_argument("--stage2_ckpt", default="checkpoints/stage2/stage2_final.pt") p.add_argument("--judge_backend", default="stub") + p.add_argument("--max_per_cat", type=int, default=0, + help="cap samples per category for eval (0 = use all)") p.add_argument("--report_dir", default="reports") args = p.parse_args() run_eval(args) diff --git a/tests/test_parse_answer.py b/tests/test_parse_answer.py index 561f616..160c9ce 100644 --- a/tests/test_parse_answer.py +++ b/tests/test_parse_answer.py @@ -45,6 +45,34 @@ class TestParseSegments: # ok=False because no valid JSON, but segments extracted via fallback assert segs == [(100, 150), (200, 250)] + def test_anomaly_regions_schema(self): + # the schema data/instruct.py actually emits (nested, start/end objects) + text = '{"anomaly_regions": [{"start": 10, "end": 93}, {"start": 245, "end": 286}], "n_regions": 2}' + segs, ok = parse_anomaly_segments(text) + assert ok is True + assert segs == [(10, 93), (245, 286)] + + def test_anomaly_regions_in_prose(self): + text = 'Based on the series: {"anomaly_regions": [{"start": 5, "end": 8}]} done.' + segs, ok = parse_anomaly_segments(text) + assert ok is True + assert segs == [(5, 8)] + + def test_segments_with_start_end_objects(self): + # also accept the {start,end} object form under the 'segments' key + text = '{"segments": [{"start": 1, "end": 4}]}' + segs, ok = parse_anomaly_segments(text) + assert ok is True + assert segs == [(1, 4)] + + def test_balanced_json_handles_nested(self): + from tsmm.eval.parse_answer import _extract_balanced_json + text = 'x {"a": {"b": 1}, "c": [2,3]} y {"d": 4}' + blobs = _extract_balanced_json(text) + # outer nested object + the flat one, but NOT the inner {"b":1} alone + assert any('"a"' in b and '"b"' in b for b in blobs) + assert '{"d": 4}' in blobs + def test_empty_segments_list(self): text = '{"segments": [], "type": "none"}' segs, ok = parse_anomaly_segments(text)