feat(m2): Collator — JSONL → wrapper raw batch (T2.5)
- src/tsmm/data/collator.py: Collator maps JSONL sample dicts to the wrapper
batch contract {series [B,max_T,max_C], attributes, timestamps, events,
question, answer}. Variable-T pad/truncate, variable-C pad, NaN→fill.
Variable-text padding/attention_mask deferred to the wrapper's per-sample
splice + max_seq padding.
- tests/test_collator.py: 11 tests. End-to-end Collator→Wrapper fwd+bwd+generate
verified (loss finite, peak 2.32 GB, generate emits text). 91 tests passing.
This commit is contained in:
@@ -0,0 +1,103 @@
|
||||
"""Collator (T2.5): JSONL sample dicts → wrapper raw batch.
|
||||
|
||||
Input : a list of sample dicts (the gen_synthetic / real_bench JSONL schema):
|
||||
{
|
||||
"series": [[T, C] floats], # may contain NaN (missing)
|
||||
"attributes": [{"name","unit","freq"}],
|
||||
"timestamps": [T ints],
|
||||
"events": [{"t","text"}],
|
||||
"instruction": str,
|
||||
"answer": str | None,
|
||||
"labels": [T ints],
|
||||
}
|
||||
|
||||
Output : the wrapper's raw batch contract:
|
||||
{
|
||||
"series": [B, max_T, max_C] float tensor (NaN→nan_fill, T pad/trunc),
|
||||
"attributes": list[str], # descriptive channel text per sample
|
||||
"timestamps": list[str], # range text per sample
|
||||
"events": list[str], # event text per sample
|
||||
"question": list[str], # from sample["instruction"]
|
||||
"answer": list[str] | None,
|
||||
}
|
||||
|
||||
Variable-length text (token padding / attention_mask) is handled downstream by
|
||||
the wrapper's per-sample splice + max_seq padding, so the collator only needs to
|
||||
produce the per-sample strings.
|
||||
|
||||
Design ref: §6.1 (``data/collator.py``).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def _attrs_to_text(attrs: List[dict]) -> str:
|
||||
parts = []
|
||||
for a in attrs:
|
||||
parts.append(f"{a.get('name','?')}({a.get('unit','')},{a.get('freq','')})")
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def _events_to_text(events: List[dict]) -> str:
|
||||
if not events:
|
||||
return ""
|
||||
parts = [f"t={e.get('t')}: {e.get('text','')}" for e in events]
|
||||
return "; ".join(parts)
|
||||
|
||||
|
||||
def _timestamps_to_text(ts: List[int]) -> str:
|
||||
if not ts:
|
||||
return ""
|
||||
if len(ts) == 1:
|
||||
return f"t={ts[0]}"
|
||||
return f"t={ts[0]}..{ts[-1]} ({len(ts)} steps)"
|
||||
|
||||
|
||||
class Collator:
|
||||
def __init__(self, max_T: int = 512, nan_fill: float = 0.0) -> None:
|
||||
self.max_T = max_T
|
||||
self.nan_fill = nan_fill
|
||||
|
||||
def _stack_series(self, samples: List[dict]) -> torch.Tensor:
|
||||
tensors = []
|
||||
for s in samples:
|
||||
arr = torch.tensor(s["series"], dtype=torch.float32) # [T, C]
|
||||
T, C = arr.shape
|
||||
# truncate / pad T to max_T
|
||||
if T >= self.max_T:
|
||||
arr = arr[: self.max_T]
|
||||
else:
|
||||
pad = torch.zeros(self.max_T - T, C)
|
||||
arr = torch.cat([arr, pad], dim=0)
|
||||
tensors.append((arr, C))
|
||||
max_C = max(c for _, c in tensors)
|
||||
B = len(tensors)
|
||||
out = torch.zeros(B, self.max_T, max_C, dtype=torch.float32)
|
||||
for i, (arr, C) in enumerate(tensors):
|
||||
out[i, :, :C] = arr[:, :max_C]
|
||||
# replace NaN (missing markers) with fill
|
||||
out = torch.nan_to_num(out, nan=self.nan_fill)
|
||||
return out
|
||||
|
||||
def __call__(self, samples: List[dict]) -> dict:
|
||||
series = self._stack_series(samples)
|
||||
attributes = [_attrs_to_text(s.get("attributes", [])) for s in samples]
|
||||
timestamps = [_timestamps_to_text(s.get("timestamps", [])) for s in samples]
|
||||
events = [_events_to_text(s.get("events", [])) for s in samples]
|
||||
question = [s.get("instruction", "") for s in samples]
|
||||
answers = [s.get("answer") for s in samples]
|
||||
if all(a is None for a in answers):
|
||||
answer: Optional[List[str]] = None
|
||||
else:
|
||||
answer = ["" if a is None else a for a in answers]
|
||||
return {
|
||||
"series": series,
|
||||
"attributes": attributes,
|
||||
"timestamps": timestamps,
|
||||
"events": events,
|
||||
"question": question,
|
||||
"answer": answer,
|
||||
}
|
||||
Reference in New Issue
Block a user