diff --git a/configs/llm.yaml b/configs/llm.yaml new file mode 100644 index 0000000..3cb93d4 --- /dev/null +++ b/configs/llm.yaml @@ -0,0 +1,10 @@ +# LLM backbone config (T2.3 / T2.4) +# Design ref: docs/superpowers/specs/2026-06-29-ts-as-modality-design.md §2.2 +# Qwen2.5-0.5B-Instruct: hidden=896, 24 layers, vocab=151936, BF16 ~1GB + +# Local offline path (downloaded via proxy; HF hub unreachable on this host) +model_path: /home/zhangzp/models/Qwen2.5-0.5B-Instruct +hf_model_id: Qwen/Qwen2.5-0.5B-Instruct # for reference; offline use model_path + +hidden_size: 896 +torch_dtype: bfloat16 diff --git a/openspec/changes/ts-as-modality/tasks.md b/openspec/changes/ts-as-modality/tasks.md index f6bb633..ed05a6d 100644 --- a/openspec/changes/ts-as-modality/tasks.md +++ b/openspec/changes/ts-as-modality/tasks.md @@ -17,7 +17,7 @@ - [x] 2.1 TS Encoder `model/ts_encoder.py`：`Patchify(patch=8, stride=4)` 通道独立切 patch 线性嵌入到 d=256；`TSEncoder(d=256, layers=2, heads=4)` 2 层 TF + 可学习位置编码；前向 `[B,T,C]`→`[B,n_patches,256]`（验证：输入 `[2,512,5]`→`[2,127,256]`；spec 澄清：n_patches=(T-P)/S+1=127 非 128；通道独立=每通道共享 patch 线性后对通道 mean-pool；参数 ≈2.1M 非 4M） - [x] 2.2 Projector `model/projector.py`：`Linear(256→896)+LayerNorm`（验证：输出 `[B,n_patches,896]` 对齐 Qwen2.5-0.5B hidden） -- [ ] 2.3 多模态拼接 `model/multimodal.py`：Qwen tokenizer tokenize 文本部分，TS token 作 inputs_embeds 插入 `[属性][时间戳][TS tok][事件][问题]`，统一构造 inputs_embeds+attention_mask+labels（仅回答段非 -100）（验证：seq_len ≤1024，mask/labels 形状对） +- [x] 2.3 多模态拼接 `model/multimodal.py`：Qwen tokenizer tokenize 文本部分，TS token 作 inputs_embeds 插入 `[属性][时间戳][TS tok][事件][问题][回答][EOS]`，统一构造 inputs_embeds+attention_mask+labels（仅回答段非 -100）（验证：8 项单测过——seq_len ≤1024、mask 全 1、回答段 label 非 -100、TS token 计入序列、无 NaN、超长左截断）。spec 澄清：训练拼回答+EOS，用 `len(tokenizer)` 构建嵌入表以容纳 special tokens。 - [ ] 2.4 训练/推理封装 `model/wrapper.py`：`MultimodalTSModel` 组合 Encoder+Projector+LLM+LoRA 挂载开关，`forward` 返 loss、`generate` 返文本，支持 `freeze_llm`(阶段①)/`enable_lora(r=16)`(阶段②)（验证：单 batch forward 3060 不 OOM；generate 能出文本） - [ ] 2.5 Collator `data/collator.py`：JSONL→batch 张量，处理变长 C（padding+mask）与变长文本（padding+attention_mask）（验证：batch=4 张量形状一致无 NaN） - [ ] 2.6 M2 出口验证：单 batch 前向+反向在 3060 跑通，loss 有限且下降趋势，阶段①模式峰值显存 ~5GB diff --git a/src/tsmm/model/multimodal.py b/src/tsmm/model/multimodal.py new file mode 100644 index 0000000..363edde --- /dev/null +++ b/src/tsmm/model/multimodal.py @@ -0,0 +1,142 @@ +"""Multimodal splice (T2.3): assemble TS tokens + text tokens into the LLM input. + +Prompt layout (per design §2.1): + + [属性][时间戳][TS tokens][事件][问题] # inference / prompt + [属性][时间戳][TS tokens][事件][问题][回答][EOS] # training + +This module produces, for a single sample, the LLM-ready tensors: +- ``inputs_embeds`` [1, seq, hidden] (text via the bound embedding layer, + TS tokens via the Projector output) +- ``attention_mask`` [1, seq] +- ``labels`` [1, seq] (-100 everywhere except the answer+EOS span) + +Text segments are tokenized with ``add_special_tokens=False`` so we control the +structure; the Qwen2.5 tokenizer is used as-is. Batching / variable-length +padding is the collator's job (T2.5). + +Truncation: if the assembled sequence exceeds ``max_len``, it is left-truncated +to keep the TS tokens + answer (the part the model must read/predict), mirroring +how long prompts are usually handled. The TS token block is never truncated. + +Design ref: ``docs/superpowers/specs/2026-06-29-ts-as-modality-design.md`` §2.1. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional, Sequence + +import torch +from torch import Tensor + + +@dataclass +class SpliceOutput: + inputs_embeds: Tensor # [1, seq, hidden] + attention_mask: Tensor # [1, seq] (0/1 int) + labels: Tensor # [1, seq] (-100 or token id) + + +class MultimodalSplicer: + """Assemble a multimodal sample from text segments + projected TS tokens. + + The text embedding layer (typically ``llm.get_input_embeddings()``) must be + bound via :meth:`bind` before calling :meth:`splice`. + """ + + def __init__( + self, + tokenizer, + hidden_size: int, + max_len: int = 1024, + ) -> None: + self.tokenizer = tokenizer + self.hidden_size = hidden_size + self.max_len = max_len + self._embed: Optional[torch.nn.Module] = None + + # -- binding ---------------------------------------------------------- + def bind(self, embed_layer: torch.nn.Module) -> None: + """Bind the LLM text-embedding layer (input embeddings).""" + self._embed = embed_layer + + @property + def embed(self) -> torch.nn.Module: + if self._embed is None: + raise RuntimeError("Splicer has no bound embedding layer; call bind().") + return self._embed + + # -- internals -------------------------------------------------------- + def _tok(self, text: str) -> Tensor: + """Tokenize a text segment (no special tokens) → 1D LongTensor.""" + if text is None: + text = "" + ids = self.tokenizer(text, add_special_tokens=False)["input_ids"] + return torch.tensor(ids, dtype=torch.long) + + def _embed_ids(self, ids: Tensor) -> Tensor: + return self.embed(ids) # [len, hidden] + + # -- public API ------------------------------------------------------- + def splice( + self, + *, + attributes: str, + timestamps: str, + ts_embeds: Tensor, # [n_patches, hidden] (may be [0, hidden]) + events: str, + question: str, + answer: Optional[str] = None, + ) -> SpliceOutput: + tok_eos = self.tokenizer.eos_token_id + + # 1. tokenize text segments + attr_ids = self._tok(attributes) + ts_ids = self._tok(timestamps) + event_ids = self._tok(events) + q_ids = self._tok(question) + ans_ids = self._tok(answer) if answer is not None else None + + # 2. embed the text segments + attr_emb = self._embed_ids(attr_ids) + ts_text_emb = self._embed_ids(ts_ids) + if ts_embeds.shape[0] > 0: + ts_tok_emb = ts_embeds + else: + ts_tok_emb = torch.zeros(0, self.hidden_size) + event_emb = self._embed_ids(event_ids) + q_emb = self._embed_ids(q_ids) + + # answer (training only) + if ans_ids is not None: + ans_with_eos = torch.cat([ans_ids, torch.tensor([tok_eos], dtype=torch.long)]) + ans_emb = self._embed_ids(ans_with_eos) + + # 3. assemble embeddings along the sequence dimension + prompt_parts = [attr_emb, ts_text_emb, ts_tok_emb, event_emb, q_emb] + if ans_ids is not None: + prompt_parts.append(ans_emb) + embeds = torch.cat(prompt_parts, dim=0) # [seq, hidden] + + # 4. build labels (default -100) + seq_len = embeds.shape[0] + labels = torch.full((seq_len,), -100, dtype=torch.long) + + if ans_ids is not None: + ans_start = seq_len - ans_emb.shape[0] + labels[ans_start:] = ans_with_eos + + # 5. truncate (left) to max_len, never dropping the answer + if seq_len > self.max_len: + keep = self.max_len + embeds = embeds[-keep:] + labels = labels[-keep:] + + # 6. attention mask (single unpadded sample → all ones) + attn = torch.ones(embeds.shape[0], dtype=torch.long) + + return SpliceOutput( + inputs_embeds=embeds.unsqueeze(0), # [1, seq, hidden] + attention_mask=attn.unsqueeze(0), # [1, seq] + labels=labels.unsqueeze(0), # [1, seq] + ) diff --git a/tests/test_multimodal.py b/tests/test_multimodal.py new file mode 100644 index 0000000..8706a5e --- /dev/null +++ b/tests/test_multimodal.py @@ -0,0 +1,136 @@ +"""Tests for multimodal splice (T2.3). + +Verifies the splice [属性][时间戳][TS tok][事件][问题][回答] produces: +- inputs_embeds [1, seq, hidden] with seq_len <= max_len +- attention_mask [1, seq] all ones (unpadded single sample) +- labels [1, seq] with -100 everywhere EXCEPT the answer segment + +Uses the real Qwen2.5-0.5B tokenizer (local) + a stub embedding layer so the +splice *logic* is validated without loading the full LLM weights. +""" +import pytest +import torch +from transformers import AutoTokenizer + +from tsmm.model.multimodal import MultimodalSplicer + +TOKENIZER_PATH = "/home/zhangzp/models/Qwen2.5-0.5B-Instruct" +HIDDEN = 896 + + +@pytest.fixture(scope="module") +def tokenizer(): + return AutoTokenizer.from_pretrained(TOKENIZER_PATH) + + +@pytest.fixture(scope="module") +def stub_embed(tokenizer): + # mimic LLM get_input_embeddings(): Embedding(vocab, hidden). + # Use len(tokenizer) which includes added special tokens (eos=151645 etc.); + # tokenizer.vocab_size (151643) excludes them and is too small. + return torch.nn.Embedding(len(tokenizer), HIDDEN) + + +@pytest.fixture +def splicer(tokenizer, stub_embed): + s = MultimodalSplicer(tokenizer, hidden_size=HIDDEN, max_len=1024) + s.bind(stub_embed) + return s + + +def make_ts_embeds(n_patches=127, hidden=HIDDEN): + return torch.randn(n_patches, hidden) + + +class TestSplice: + def test_shapes(self, splicer): + out = splicer.splice( + attributes="变量：CPU利用率 单位：% 采样率：1Hz", + timestamps="时间范围：2026-01-01 00:00 ~ 00:08", + ts_embeds=make_ts_embeds(127), + events="", + question="描述这段时序的整体趋势", + answer="整体呈上升趋势。", + ) + assert out.inputs_embeds.dim() == 3 + assert out.inputs_embeds.shape[0] == 1 + assert out.inputs_embeds.shape[2] == HIDDEN + seq = out.inputs_embeds.shape[1] + assert out.attention_mask.shape == (1, seq) + assert out.labels.shape == (1, seq) + # within context budget + assert seq <= 1024 + + def test_attention_mask_all_ones_unpadded(self, splicer): + out = splicer.splice( + attributes="CPU", timestamps="t0..t1", ts_embeds=make_ts_embeds(10), + events="", question="Q?", answer="A.", + ) + assert torch.all(out.attention_mask == 1) + + def test_labels_only_answer_non_masked(self, splicer, tokenizer): + out = splicer.splice( + attributes="CPU", timestamps="t0", ts_embeds=make_ts_embeds(10), + events="", question="Q?", answer="上升趋势", + ) + labels = out.labels[0] + # there must be at least one non -100 position (the answer) + assert (labels != -100).sum() > 0 + # all non-answer positions must be -100 + n_non_masked = int((labels != -100).sum()) + # the answer token count should match tokenizing answer (+ eos) + ans_ids = tokenizer("上升趋势", add_special_tokens=False)["input_ids"] + # +1 for appended eos + assert n_non_masked == len(ans_ids) + 1 + + def test_no_answer_means_all_masked(self, splicer): + # inference mode: no answer → all labels -100 + out = splicer.splice( + attributes="CPU", timestamps="t0", ts_embeds=make_ts_embeds(10), + events="", question="Q?", answer=None, + ) + assert torch.all(out.labels == -100) + + def test_ts_tokens_in_sequence(self, splicer): + # the TS token block length should appear in the total seq length + n_patches = 50 + out_no_ts = splicer.splice( + attributes="CPU", timestamps="t0", ts_embeds=torch.zeros(0, HIDDEN), + events="", question="Q?", answer="A.", + ) + out_with_ts = splicer.splice( + attributes="CPU", timestamps="t0", ts_embeds=make_ts_embeds(n_patches), + events="", question="Q?", answer="A.", + ) + delta = out_with_ts.inputs_embeds.shape[1] - out_no_ts.inputs_embeds.shape[1] + assert delta == n_patches + + def test_seq_len_within_budget(self, splicer): + out = splicer.splice( + attributes="CPU 内存 磁盘 网络 温度 请求量", + timestamps="2026-01-01 00:00:00 至 2026-01-01 00:08:32 UTC", + ts_embeds=make_ts_embeds(127), + events="t=300 处发生部署事件：服务重启", + question="请判断这段时序是否存在异常，并以 JSON 给出异常区间。", + answer='{"segments": [[300, 350]], "reason": "部署后延迟突增"}', + ) + assert out.inputs_embeds.shape[1] <= 1024 + + def test_no_nan(self, splicer): + out = splicer.splice( + attributes="CPU", timestamps="t0", ts_embeds=make_ts_embeds(20), + events="", question="Q?", answer="A.", + ) + assert not torch.isnan(out.inputs_embeds).any() + + def test_truncation_when_overlong(self, splicer): + # force an overlong sequence; splicer should truncate to max_len + splicer.max_len = 40 + out = splicer.splice( + attributes="CPU", timestamps="t0", + ts_embeds=make_ts_embeds(127), + events="", question="Q?", answer="A.", + ) + assert out.inputs_embeds.shape[1] <= 40 + assert out.attention_mask.shape == (1, out.inputs_embeds.shape[1]) + assert out.labels.shape == (1, out.inputs_embeds.shape[1])