feat(m2): Collator — JSONL → wrapper raw batch (T2.5)

- src/tsmm/data/collator.py: Collator maps JSONL sample dicts to the wrapper
  batch contract {series [B,max_T,max_C], attributes, timestamps, events,
  question, answer}. Variable-T pad/truncate, variable-C pad, NaN→fill.
  Variable-text padding/attention_mask deferred to the wrapper's per-sample
  splice + max_seq padding.
- tests/test_collator.py: 11 tests. End-to-end Collator→Wrapper fwd+bwd+generate
  verified (loss finite, peak 2.32 GB, generate emits text). 91 tests passing.
This commit is contained in:
张宗平
2026-06-30 02:14:47 +00:00
parent aec8fd5ef1
commit 7e1df76e3e
3 changed files with 201 additions and 1 deletions
+103
View File
@@ -0,0 +1,103 @@
"""Collator (T2.5): JSONL sample dicts → wrapper raw batch.
Input : a list of sample dicts (the gen_synthetic / real_bench JSONL schema):
{
"series": [[T, C] floats], # may contain NaN (missing)
"attributes": [{"name","unit","freq"}],
"timestamps": [T ints],
"events": [{"t","text"}],
"instruction": str,
"answer": str | None,
"labels": [T ints],
}
Output : the wrapper's raw batch contract:
{
"series": [B, max_T, max_C] float tensor (NaN→nan_fill, T pad/trunc),
"attributes": list[str], # descriptive channel text per sample
"timestamps": list[str], # range text per sample
"events": list[str], # event text per sample
"question": list[str], # from sample["instruction"]
"answer": list[str] | None,
}
Variable-length text (token padding / attention_mask) is handled downstream by
the wrapper's per-sample splice + max_seq padding, so the collator only needs to
produce the per-sample strings.
Design ref: §6.1 (``data/collator.py``).
"""
from __future__ import annotations
from typing import Any, List, Optional
import torch
def _attrs_to_text(attrs: List[dict]) -> str:
parts = []
for a in attrs:
parts.append(f"{a.get('name','?')}({a.get('unit','')},{a.get('freq','')})")
return " ".join(parts)
def _events_to_text(events: List[dict]) -> str:
if not events:
return ""
parts = [f"t={e.get('t')}: {e.get('text','')}" for e in events]
return "; ".join(parts)
def _timestamps_to_text(ts: List[int]) -> str:
if not ts:
return ""
if len(ts) == 1:
return f"t={ts[0]}"
return f"t={ts[0]}..{ts[-1]} ({len(ts)} steps)"
class Collator:
def __init__(self, max_T: int = 512, nan_fill: float = 0.0) -> None:
self.max_T = max_T
self.nan_fill = nan_fill
def _stack_series(self, samples: List[dict]) -> torch.Tensor:
tensors = []
for s in samples:
arr = torch.tensor(s["series"], dtype=torch.float32) # [T, C]
T, C = arr.shape
# truncate / pad T to max_T
if T >= self.max_T:
arr = arr[: self.max_T]
else:
pad = torch.zeros(self.max_T - T, C)
arr = torch.cat([arr, pad], dim=0)
tensors.append((arr, C))
max_C = max(c for _, c in tensors)
B = len(tensors)
out = torch.zeros(B, self.max_T, max_C, dtype=torch.float32)
for i, (arr, C) in enumerate(tensors):
out[i, :, :C] = arr[:, :max_C]
# replace NaN (missing markers) with fill
out = torch.nan_to_num(out, nan=self.nan_fill)
return out
def __call__(self, samples: List[dict]) -> dict:
series = self._stack_series(samples)
attributes = [_attrs_to_text(s.get("attributes", [])) for s in samples]
timestamps = [_timestamps_to_text(s.get("timestamps", [])) for s in samples]
events = [_events_to_text(s.get("events", [])) for s in samples]
question = [s.get("instruction", "") for s in samples]
answers = [s.get("answer") for s in samples]
if all(a is None for a in answers):
answer: Optional[List[str]] = None
else:
answer = ["" if a is None else a for a in answers]
return {
"series": series,
"attributes": attributes,
"timestamps": timestamps,
"events": events,
"question": question,
"answer": answer,
}