feat(m4/m5): T4.2 instruct-check + QA judge table in eval report
- eval/instruct_check.py (T4.2): per-category (6 types) parse_rate + rule_acc + stub-judge mean + usability; pass bars anomaly-JSON>80%, overall usability>70%. - eval/report.py: wire eval_qa_task across all categories so the report now covers design §6.5 问答均分 (QA judge 0-5 per category per predictor), not just anomaly-detection metrics. Fix latent bug: describe rule judge expects a dict ref, so parse the JSON ref_answer before passing. - Verified trivial-only smoke renders both tables; all_report VUS-PR=0.26 (not inflated) vs PA-F1~1.0 confirms PA is control-only.
This commit is contained in:
张宗平
@@ -0,0 +1,213 @@
|
||||
"""Instruction-following check (T4.2).
|
||||
|
||||
Samples N held-out items per task category from ``eval_synth.jsonl`` and runs the
|
||||
trained model, then reports per-category:
|
||||
|
||||
- **parse_rate**: for the structured categories (anomaly / describe / forecast)
|
||||
the fraction of answers whose JSON parses; for free-text categories
|
||||
(root_cause / event / compare) the fraction of non-empty answers.
|
||||
- **rule_acc**: rule-judge score where the category is rule-checkable.
|
||||
- **judge_mean**: stub LLM-judge 0-5 mean (token-overlap heuristic, offline).
|
||||
|
||||
Design T4.2 pass bars (§4.3 M4 出口验证):
|
||||
- anomaly JSON parse-success rate > 80%
|
||||
- 6-category overall usability (parse or non-empty + judge>0) > 70%
|
||||
|
||||
Heavy (GPU, model.generate) — imports the wrapper lazily.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
CATEGORIES = ["anomaly", "describe", "forecast", "root_cause", "event", "compare"]
|
||||
|
||||
|
||||
def load_by_category(path: str, per_cat: int, seed: int = 0) -> Dict[str, List[dict]]:
|
||||
buckets: Dict[str, List[dict]] = defaultdict(list)
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
s = json.loads(line)
|
||||
buckets[s.get("category", "?")].append(s)
|
||||
rng = random.Random(seed)
|
||||
out = {}
|
||||
for cat in CATEGORIES:
|
||||
pool = buckets.get(cat, [])
|
||||
rng.shuffle(pool)
|
||||
out[cat] = pool[:per_cat]
|
||||
return out
|
||||
|
||||
|
||||
def _build_model(stage1_ckpt=None, stage2_ckpt=None):
|
||||
import torch
|
||||
from ..model.ts_encoder import TSEncoder
|
||||
from ..model.projector import Projector
|
||||
from ..model.wrapper import MultimodalTSModel
|
||||
from ..data.collator import Collator
|
||||
|
||||
llm = os.environ.get("TSMM_LLM_PATH", "/home/zhangzp/models/Qwen2.5-0.5B-Instruct")
|
||||
m = MultimodalTSModel(llm, TSEncoder(), Projector(), dtype=torch.bfloat16).cuda()
|
||||
ckpt = stage2_ckpt or stage1_ckpt
|
||||
if ckpt and os.path.exists(ckpt):
|
||||
st = torch.load(ckpt, map_location="cpu")
|
||||
m.encoder.load_state_dict(st["encoder"])
|
||||
m.projector.load_state_dict(st["projector"])
|
||||
print(f"[instruct_check] loaded {ckpt}")
|
||||
lora_dir = os.path.join(os.path.dirname(ckpt), "lora_adapter")
|
||||
if stage2_ckpt and os.path.exists(lora_dir):
|
||||
from peft import PeftModel
|
||||
m.llm = PeftModel.from_pretrained(m.llm, lora_dir)
|
||||
print(f"[instruct_check] loaded lora {lora_dir}")
|
||||
m.eval()
|
||||
return m, Collator(max_T=512)
|
||||
|
||||
|
||||
def _generate(m, col, sample) -> str:
|
||||
import torch
|
||||
batch = col([sample])
|
||||
with torch.no_grad():
|
||||
txt = m.generate(batch, max_new_tokens=96)
|
||||
return txt[0] if txt else ""
|
||||
|
||||
|
||||
def evaluate(samples_by_cat, model, col, judge_backend="stub") -> Dict[str, Dict]:
|
||||
from .parse_answer import parse_anomaly_segments
|
||||
from .qa_judge import RuleJudge, LLMJudge, _extract_json
|
||||
|
||||
rule_j = RuleJudge()
|
||||
llm_j = LLMJudge(backend=judge_backend)
|
||||
report: Dict[str, Dict] = {}
|
||||
|
||||
for cat, samples in samples_by_cat.items():
|
||||
if not samples:
|
||||
continue
|
||||
n = len(samples)
|
||||
parsed = 0 # structured-parse success OR non-empty prose
|
||||
nonempty = 0
|
||||
rule_scores: List[float] = []
|
||||
judge_scores: List[float] = []
|
||||
examples: List[Dict] = []
|
||||
|
||||
for s in samples:
|
||||
pred = _generate(model, col, s)
|
||||
ref = s.get("answer", "")
|
||||
instr = s.get("instruction") or s.get("question") or ""
|
||||
|
||||
ok = False
|
||||
if cat == "anomaly":
|
||||
_, ok = parse_anomaly_segments(pred)
|
||||
elif cat in ("describe", "forecast"):
|
||||
ok = _extract_json(pred) is not None
|
||||
else: # free-text
|
||||
ok = bool(pred.strip())
|
||||
if ok:
|
||||
parsed += 1
|
||||
if pred.strip():
|
||||
nonempty += 1
|
||||
|
||||
r = rule_j.judge(
|
||||
cat, pred,
|
||||
ref_segs=[(seg.get("start", 0), seg.get("end", 0))
|
||||
for seg in s.get("segments", [])],
|
||||
T=len(s.get("labels", [])) or 1,
|
||||
ref_answer=ref if cat == "describe" else "",
|
||||
ref_forecast=_forecast_list(ref) if cat == "forecast" else [],
|
||||
)
|
||||
if r.get("score") is not None:
|
||||
rule_scores.append(r["score"])
|
||||
|
||||
j = llm_j.score(instr, ref, pred)
|
||||
judge_scores.append(j["score"])
|
||||
|
||||
if len(examples) < 2:
|
||||
examples.append({
|
||||
"pred": (pred or "")[:160],
|
||||
"ref": (ref or "")[:160],
|
||||
"parsed": ok,
|
||||
})
|
||||
|
||||
# usability = parsed (structured-or-nonempty) AND judge thinks there's
|
||||
# some content (judge>0). Conservative per-design ">70% usable".
|
||||
usable = sum(1 for sc in judge_scores if sc > 0)
|
||||
report[cat] = {
|
||||
"n": n,
|
||||
"parse_rate": parsed / n,
|
||||
"nonempty_rate": nonempty / n,
|
||||
"rule_acc": float(np.mean(rule_scores)) if rule_scores else None,
|
||||
"judge_mean": float(np.mean(judge_scores)),
|
||||
"usability": usable / n,
|
||||
"examples": examples,
|
||||
}
|
||||
return report
|
||||
|
||||
|
||||
def _forecast_list(ref_answer):
|
||||
"""Extract the forecast value list from a forecast ref answer JSON."""
|
||||
try:
|
||||
obj = json.loads(ref_answer) if isinstance(ref_answer, str) else ref_answer
|
||||
fc = obj.get("forecast")
|
||||
if isinstance(fc, list) and fc:
|
||||
inner = fc[0]
|
||||
if isinstance(inner, list):
|
||||
return [float(x) for x in inner]
|
||||
return [float(x) for x in fc]
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> Dict:
|
||||
samples_by_cat = load_by_category(args.data, args.per_cat, args.seed)
|
||||
model, col = _build_model(args.stage1_ckpt, args.stage2_ckpt)
|
||||
rep = evaluate(samples_by_cat, model, col, args.judge_backend)
|
||||
|
||||
# headline summary
|
||||
anomaly_parse = rep.get("anomaly", {}).get("parse_rate", 0.0)
|
||||
overall_usability = float(np.mean([r["usability"] for r in rep.values()])) if rep else 0.0
|
||||
summary = {
|
||||
"anomaly_json_parse_rate": anomaly_parse,
|
||||
"anomaly_json_pass": anomaly_parse > 0.80,
|
||||
"overall_usability": overall_usability,
|
||||
"overall_usability_pass": overall_usability > 0.70,
|
||||
}
|
||||
print("\n=== T4.2 instruction-following summary ===")
|
||||
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
||||
print("\n=== per-category ===")
|
||||
for cat, r in rep.items():
|
||||
print(f"[{cat}] n={r['n']} parse={r['parse_rate']:.2f} "
|
||||
f"rule={r['rule_acc']} judge={r['judge_mean']:.2f} "
|
||||
f"usable={r['usability']:.2f}")
|
||||
for ex in r["examples"]:
|
||||
print(f" pred={ex['pred']!r}")
|
||||
return {"summary": summary, "per_category": rep}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser(description="Instruction-following check (T4.2)")
|
||||
p.add_argument("--data", default="data/eval_synth.jsonl")
|
||||
p.add_argument("--per_cat", type=int, default=10)
|
||||
p.add_argument("--seed", type=int, default=0)
|
||||
p.add_argument("--stage1_ckpt", default="checkpoints/stage1/stage1_final.pt")
|
||||
p.add_argument("--stage2_ckpt", default="checkpoints/stage2/stage2_final.pt")
|
||||
p.add_argument("--judge_backend", default="stub")
|
||||
p.add_argument("--out", default="reports/instruct_check.json")
|
||||
args = p.parse_args()
|
||||
result = run(args)
|
||||
os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
|
||||
with open(args.out, "w", encoding="utf-8") as f:
|
||||
json.dump(result, f, indent=2, ensure_ascii=False)
|
||||
print(f"\n[instruct_check] wrote {args.out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+48
-4
@@ -100,11 +100,20 @@ def eval_qa_task(samples, predictor, judge_backend="stub") -> Dict[str, Any]:
|
||||
cat = s.get("category", "root_cause")
|
||||
pred = predictor(s)
|
||||
ref = s.get("answer", "")
|
||||
# describe expects a dict ref; parse JSON if possible, else {}
|
||||
ref_for_describe = ref
|
||||
if cat == "describe":
|
||||
try:
|
||||
ref_for_describe = json.loads(ref) if isinstance(ref, str) else (ref or {})
|
||||
if not isinstance(ref_for_describe, dict):
|
||||
ref_for_describe = {}
|
||||
except (ValueError, TypeError):
|
||||
ref_for_describe = {}
|
||||
r = rule_judge.judge(cat, pred,
|
||||
ref_segs=[(seg.get("start", 0), seg.get("end", 0))
|
||||
for seg in s.get("segments", [])],
|
||||
T=len(s.get("labels", [])) or 1,
|
||||
ref_answer=ref)
|
||||
ref_answer=ref_for_describe)
|
||||
if r.get("parsed"):
|
||||
parse_ok += 1
|
||||
if r.get("score") is not None:
|
||||
@@ -189,17 +198,25 @@ def run_eval(args: argparse.Namespace) -> str:
|
||||
|
||||
# run anomaly eval per dataset × predictor
|
||||
results: Dict[str, Dict[str, Any]] = defaultdict(dict)
|
||||
qa_results: Dict[str, Dict[str, Dict[str, Any]]] = defaultdict(lambda: defaultdict(dict))
|
||||
for ds_name, path in datasets:
|
||||
samples = list(stream_jsonl(path))
|
||||
anom = [s for s in samples if s.get("category") == "anomaly"]
|
||||
by_cat: Dict[str, List[dict]] = defaultdict(list)
|
||||
for s in samples:
|
||||
by_cat[s.get("category", "?")].append(s)
|
||||
anom = by_cat.get("anomaly", [])
|
||||
for pname, pred in predictors.items():
|
||||
results[ds_name][pname] = eval_anomaly_task(anom, pred)
|
||||
# QA judge across all categories (covers design §6.5 问答均分)
|
||||
for cat, cat_samples in by_cat.items():
|
||||
qa_results[ds_name][pname][cat] = eval_qa_task(
|
||||
cat_samples, pred, judge_backend=args.judge_backend)
|
||||
|
||||
# render markdown
|
||||
return _render_report(results, datasets, predictors.keys(), args)
|
||||
return _render_report(results, qa_results, datasets, predictors.keys(), args)
|
||||
|
||||
|
||||
def _render_report(results, datasets, predictor_names, args) -> str:
|
||||
def _render_report(results, qa_results, datasets, predictor_names, args) -> str:
|
||||
today = date.today().isoformat()
|
||||
lines = [
|
||||
f"# Eval Report — {today}",
|
||||
@@ -237,6 +254,33 @@ def _render_report(results, datasets, predictor_names, args) -> str:
|
||||
lines.append("_PA = point-adjust; shown as control only, not headline._")
|
||||
lines.append("")
|
||||
|
||||
# QA judge table (covers design §6.5 success criterion 问答均分)
|
||||
qa_ds = qa_results.get(ds_name, {})
|
||||
if qa_ds:
|
||||
cats = sorted({c for pm in qa_ds.values() for c in pm})
|
||||
lines.append(f"### QA judge (stub 0-5) — {ds_name}")
|
||||
lines.append("")
|
||||
header = ["predictor"] + cats + ["mean"]
|
||||
lines.append("| " + " | ".join(header) + " |")
|
||||
lines.append("|" + "|".join(["---"] * len(header)) + "|")
|
||||
for pname in predictor_names:
|
||||
pm = qa_ds.get(pname, {})
|
||||
if not pm:
|
||||
continue
|
||||
row_vals = []
|
||||
for c in cats:
|
||||
jm = pm.get(c, {}).get("judge_mean")
|
||||
row_vals.append(f"{jm:.2f}" if jm is not None else "-")
|
||||
mean_jm = float(np.mean([pm[c]["judge_mean"]
|
||||
for c in cats
|
||||
if pm.get(c, {}).get("judge_mean") is not None])) if any(pm.get(c) for c in cats) else 0.0
|
||||
row = [pname] + row_vals + [f"{mean_jm:.2f}"]
|
||||
lines.append("| " + " | ".join(row) + " |")
|
||||
lines.append("")
|
||||
lines.append("_QA judge = offline stub token-overlap heuristic; "
|
||||
"swap `--judge_backend openai` for GPT-4o scoring._")
|
||||
lines.append("")
|
||||
|
||||
out_path = os.path.join(args.report_dir, f"eval-{today}.md")
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(lines))
|
||||
|
||||
Reference in New Issue
Block a user