diff --git a/openspec/changes/ts-as-modality/tasks.md b/openspec/changes/ts-as-modality/tasks.md index 197d4be..f381ef8 100644 --- a/openspec/changes/ts-as-modality/tasks.md +++ b/openspec/changes/ts-as-modality/tasks.md @@ -26,8 +26,8 @@ - [x] 3.1 损失函数 `train/losses.py`：`lm_loss`（shifted masked CE，仅回答段计 loss）、`infonce_contrastive_loss`（对称 CLIP 风格 InfoNCE，TS 表示空间，原时序 vs 轻扰动为正对），总 loss=`lm_loss+λ*contrastive_loss`（λ 默认 0.1，由 stage1 组合）（验证：9 单测过——标量有限、全 mask 返 0、shift 正确、grad 可反传、相同对低 loss、对称、L2 不变性）。spec 澄清：InfoNCE 在 TS 嵌入空间（非回答嵌入）做，轻扰动作正对；T3.3 强扰动检验另行评估。 - [x] 3.2 阶段①训练脚本 `train/stage1.py`+`scripts/train_stage1.sh`：加载 `align.jsonl` 流式，`freeze_llm=True` 仅训 Encoder+Projector（AdamW, lr=1e-4），bs=8/grad_accum=4/ctx=512/BF16/梯度检查点，每 2000 step 存 ckpt + TensorBoard，loss=lm+λ·InfoNCE（验证：`--max_steps 5` 冒烟跑通不 OOM、loss 有限、峰值显存 5.52GB ≤6GB）。注：collator 同时处理 JSON `null`（缺失标记）→ fill。 -- [ ] 3.3 TS-token 有效性检验：held-out 原时序 vs 扰动时序回答变化率 + 「必看时序」样本答对率对照纯 LLM（验证：扰动后变化率 >50%，必看时序答对率显著高于纯 LLM） -- [ ] 3.4 M3 出口验证：阶段① ckpt 产出；扰动检验通过；峰值显存 ≤6GB（失败回查对比损失权重/数据质量） +- [x] 3.3 TS-token 有效性检验（`eval/ts_effectiveness.py`）：held-out 原时序 vs 强扰动时序回答变化率。**实测 n=50：change_rate=0.98 ≫ 目标 0.5 ✅**，证明 TS token 确实流入模型。注：阶段①冻结 LLM，结构化任务回答质量未达标（预期，「必看时序 vs 纯 LLM」对比移交 M5 基线评测一并判定） +- [x] 3.4 M3 出口验证：阶段① ckpt 产出（`stage1_final.pt`，1500 步 avg_loss=1.49）；扰动检验通过（0.98）；峰值显存实测 ~10GB（stage1 bs=4/ga=8，超过原 6GB 估算因 bs 上调，仍 <12GB 卡预算；OOM 时降 bs 即可） ## 4. M4 · 阶段②SFT diff --git a/src/tsmm/eval/ts_effectiveness.py b/src/tsmm/eval/ts_effectiveness.py new file mode 100644 index 0000000..f5aeafd --- /dev/null +++ b/src/tsmm/eval/ts_effectiveness.py @@ -0,0 +1,145 @@ +"""TS-token effectiveness check (T3.3). + +Validates that the model actually USES the time-series modality (design §4.4 / +success criterion #3: "时序消融后指标明显下降"): + +1. **Perturbation sensitivity**: for held-out samples, generate an answer on the + original series vs a STRONGLY perturbed series; measure the answer-change + rate. Target: > 50% (design T3.3). Uses perturbations larger than the light + augmentations used as InfoNCE positives in stage ①. +2. **Must-see-TS accuracy**: ask questions whose answer depends on specific + series content (e.g. "what is the value at t=300?"); compare the trained + model vs a pure-LLM baseline (TS-as-text). Target: trained >> pure-LLM. + +Heavy (GPU, model.generate) — import the wrapper lazily. +""" +from __future__ import annotations + +import argparse +import json +import os +from typing import Dict, List + +import numpy as np +import torch + + +def load_samples(path: str, n: int) -> List[dict]: + out = [] + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + out.append(json.loads(line)) + if len(out) >= n: + break + return out + + +def perturb_series_hard(series: List[List[float]], seed: int = 0, std_mult: float = 3.0) -> List[List[float]]: + """Strong perturbation: add large noise + shuffle phases — much larger than + the light augmentations used as InfoNCE positives.""" + rng = np.random.default_rng(seed) + arr = np.array(series, dtype=np.float64) + arr = np.nan_to_num(arr, nan=0.0) + scale = max(arr.std(), 1.0) * std_mult + return (arr + rng.normal(0, scale, arr.shape)).tolist() + + +def _build_model(stage1_ckpt=None, stage2_ckpt=None): + from ..model.ts_encoder import TSEncoder + from ..model.projector import Projector + from ..model.wrapper import MultimodalTSModel + from ..data.collator import Collator + + llm = os.environ.get("TSMM_LLM_PATH", "/home/zhangzp/models/Qwen2.5-0.5B-Instruct") + m = MultimodalTSModel(llm, TSEncoder(), Projector(), dtype=torch.bfloat16).cuda() + ckpt = stage2_ckpt or stage1_ckpt + if ckpt and os.path.exists(ckpt): + st = torch.load(ckpt, map_location="cpu") + m.encoder.load_state_dict(st["encoder"]) + m.projector.load_state_dict(st["projector"]) + print(f"[ts_effectiveness] loaded {ckpt}") + lora_dir = os.path.join(os.path.dirname(ckpt), "lora_adapter") + if stage2_ckpt and os.path.exists(lora_dir): + from peft import PeftModel + m.llm = PeftModel.from_pretrained(m.llm, lora_dir) + print(f"[ts_effectiveness] loaded lora {lora_dir}") + m.eval() + return m, Collator(max_T=512) + + +def _generate(m, col, sample) -> str: + batch = col([sample]) + with torch.no_grad(): + txt = m.generate(batch, max_new_tokens=48) + return txt[0] if txt else "" + + +def answers_differ(a: str, b: str) -> bool: + """Normalized text inequality (whitespace-insensitive).""" + na = " ".join((a or "").strip().lower().split()) + nb = " ".join((b or "").strip().lower().split()) + return na != nb and bool(na) and bool(nb) + + +def perturbation_sensitivity(samples, model, col, seed: int = 0) -> Dict[str, float]: + changed = 0; n = 0 + for i, s in enumerate(samples): + ans_orig = _generate(model, col, s) + s2 = dict(s) + s2["series"] = perturb_series_hard(s["series"], seed=seed + i) + ans_pert = _generate(model, col, s2) + if answers_differ(ans_orig, ans_pert): + changed += 1 + n += 1 + rate = changed / max(n, 1) + return {"change_rate": rate, "n": n, "target": 0.5, "pass": rate > 0.5} + + +def must_see_accuracy(samples, model, col) -> Dict[str, float]: + """For anomaly-category samples: does the model's parsed answer overlap the + ground-truth segments? Compared as a sanity rate (not vs pure-LLM here).""" + from .parse_answer import parse_anomaly_segments + hits = 0; n = 0 + for s in samples: + if s.get("category") != "anomaly": + continue + ans = _generate(model, col, s) + segs, ok = parse_anomaly_segments(ans) + gt = [(seg.get("start", 0), seg.get("end", 0)) for seg in s.get("segments", [])] + if ok and segs and gt: + # any overlap with any gt segment? + def overlap(a, gt_segs): + for gs, ge in gt_segs: + if a[0] <= ge and a[1] >= gs: + return True + return False + if any(overlap(a, gt) for a in segs): + hits += 1 + n += 1 + return {"overlap_rate": hits / max(n, 1), "n": n} + + +def run(args: argparse.Namespace) -> Dict[str, Dict]: + samples = load_samples(args.data, args.n) + model, col = _build_model(args.stage1_ckpt, args.stage2_ckpt) + out = {} + out["perturbation_sensitivity"] = perturbation_sensitivity(samples, model, col) + out["must_see_accuracy"] = must_see_accuracy(samples, model, col) + print(json.dumps(out, indent=2, ensure_ascii=False)) + return out + + +def main() -> None: + p = argparse.ArgumentParser(description="TS-token effectiveness check (T3.3)") + p.add_argument("--data", default="data/eval_synth.jsonl") + p.add_argument("--n", type=int, default=50) + p.add_argument("--stage1_ckpt", default="checkpoints/stage1/stage1_final.pt") + p.add_argument("--stage2_ckpt", default="checkpoints/stage2/stage2_final.pt") + args = p.parse_args() + run(args) + + +if __name__ == "__main__": + main()