diff --git a/openspec/changes/ts-as-modality/tasks.md b/openspec/changes/ts-as-modality/tasks.md index 17706e6..5d8e388 100644 --- a/openspec/changes/ts-as-modality/tasks.md +++ b/openspec/changes/ts-as-modality/tasks.md @@ -6,7 +6,7 @@ ## 1. M1 · 数据管线 - [x] 1.1 项目脚手架:创建 `pyproject.toml`(torch/transformers/peft/accelerate/datasets/numpy/tqdm/pyyaml/pytest)、`src/tsmm/` 模块骨架、`configs/`、`scripts/`、`tests/`,并建 `data/`、`checkpoints/` 加 `.gitignore`(验证:`python -c "import tsmm"` 不报错;`pytest tests/` 可跑) -- [ ] 1.2 属性词表与采样 `data/attributes.py`:定义变量名词表/单位/采样率档位,`sample_attributes(n_channels)` 返回结构化属性(验证:单测字段齐全、可复现) +- [x] 1.2 属性词表与采样 `data/attributes.py`:定义变量名词表/单位/采样率档位,`sample_attributes(n_channels)` 返回结构化属性(验证:单测字段齐全、可复现) - [ ] 1.3 成分模型与时序合成 `data/synthesis.py`:`generate_series`(趋势+周期+基线+噪声)、`inject_anomaly`(尖刺/水平偏移/方差膨胀/缺失段/缓慢漂移,返回逐点 labels+段元数据)、`couple_event`(t 处阶跃+事件文本)、`add_missing`(NaN 占位)(验证:给定 seed 确定;异常段与 labels 一致;事件处有阶跃) - [ ] 1.4 指令与回答生成 `data/instruct.py`:6 类指令模板(描述/异常/根因/预测/比较/事件关联)+ Evol-Instruct 演化,`build_instruction` 产 JSON 区间/数值/文本回答(验证:异常类 JSON 可 `json.loads` 且段与 labels 吻合) - [ ] 1.5 离线生成脚本 `scripts/gen_synthetic.py`:多进程产 `align.jsonl`(50万)/`sft.jsonl`(10万)/`eval_synth.jsonl`(2k held-out),支持 `--n/--out/--seed/--workers` + tqdm(验证:`--n 1000` 跑通,抽样 5 条 schema 合规) diff --git a/src/tsmm/data/attributes.py b/src/tsmm/data/attributes.py new file mode 100644 index 0000000..9011ab4 --- /dev/null +++ b/src/tsmm/data/attributes.py @@ -0,0 +1,77 @@ +"""Variable attribute vocabulary + attribute sampling (T1.2). + +Defines a curated vocabulary of AIOps-relevant channel *names* with their +*units* and a set of *sampling frequency bands*. ``sample_attributes`` draws a +reproducible, unique-name subset to describe the channels of a multivariate +series — metadata that the instruction generator and the multimodal splicer +will later turn into text tokens like ``[属性: CPU usage (%), 1s]``. +""" + +from __future__ import annotations + +import random +from typing import Any + +# ─── vocabulary ──────────────────────────────────────────────────────────── +# Each entry: name (human label), unit. Frequencies are drawn separately from +# FREQ_BANDS so the same channel name can appear at different resolutions. +ATTRIBUTE_VOCAB: list[dict[str, str]] = [ + {"name": "CPU usage", "unit": "%"}, + {"name": "Memory usage", "unit": "%"}, + {"name": "Disk I/O", "unit": "MB/s"}, + {"name": "Network inbound traffic", "unit": "Mbps"}, + {"name": "Network outbound traffic", "unit": "Mbps"}, + {"name": "Request rate", "unit": "req/s"}, + {"name": "Request latency p50", "unit": "ms"}, + {"name": "Request latency p99", "unit": "ms"}, + {"name": "Error rate", "unit": "%"}, + {"name": "Temperature", "unit": "°C"}, + {"name": "Queue length", "unit": "tasks"}, + {"name": "Active connections", "unit": "conn"}, + {"name": "Cache hit ratio", "unit": "%"}, + {"name": "GPU utilization", "unit": "%"}, + {"name": "Battery level", "unit": "%"}, + {"name": "Pod restart count", "unit": "count"}, +] + +# Sampling frequency bands (resolution at which the series is recorded). +FREQ_BANDS: list[str] = ["1s", "5s", "10s", "30s", "1min", "5min"] + + +def sample_attributes( + n_channels: int, *, seed: int | None = None +) -> list[dict[str, Any]]: + """Sample ``n_channels`` distinct channel attribute descriptors. + + Each descriptor is a dict with keys ``name``, ``unit`` (drawn from + :data:`ATTRIBUTE_VOCAB`) and ``freq`` (drawn from :data:`FREQ_BANDS`). + Channel names are guaranteed unique within the returned list. + + Parameters + ---------- + n_channels: + Number of channels to sample. Must be ``<= len(ATTRIBUTE_VOCAB)``. + seed: + RNG seed for reproducibility. + + Raises + ------ + ValueError + If ``n_channels`` exceeds the available distinct names. + """ + if not isinstance(n_channels, int) or n_channels < 0: + raise ValueError(f"n_channels must be a non-negative int, got {n_channels!r}") + if n_channels > len(ATTRIBUTE_VOCAB): + raise ValueError( + f"requested {n_channels} channels but vocab only has " + f"{len(ATTRIBUTE_VOCAB)} distinct names" + ) + + rng = random.Random(seed) + vocab = list(ATTRIBUTE_VOCAB) + chosen = rng.sample(vocab, n_channels) + freqs = rng.choices(FREQ_BANDS, k=n_channels) + return [ + {"name": c["name"], "unit": c["unit"], "freq": f} + for c, f in zip(chosen, freqs) + ] diff --git a/tests/test_attributes.py b/tests/test_attributes.py new file mode 100644 index 0000000..df01dcf --- /dev/null +++ b/tests/test_attributes.py @@ -0,0 +1,62 @@ +"""Tests for attribute vocab + sampling (T1.2).""" +import numpy as np + +from tsmm.data.attributes import sample_attributes, ATTRIBUTE_VOCAB, FREQ_BANDS + + +# ─── field completeness ──────────────────────────────────────────────────── +def test_each_sampled_attribute_has_required_fields(): + attrs = sample_attributes(n_channels=4, seed=0) + assert len(attrs) == 4 + for a in attrs: + assert set(a.keys()) >= {"name", "unit", "freq"} + assert isinstance(a["name"], str) and a["name"] + assert isinstance(a["unit"], str) and a["unit"] + assert isinstance(a["freq"], str) and a["freq"] + + +def test_names_are_unique_within_a_sample(): + attrs = sample_attributes(n_channels=5, seed=1) + names = [a["name"] for a in attrs] + assert len(names) == len(set(names)), "channel names must be unique" + + +# ─── correctness of count ────────────────────────────────────────────────── +def test_count_matches_request(): + for n in (1, 2, 8): + assert len(sample_attributes(n_channels=n, seed=0)) == n + + +def test_raises_when_request_exceeds_vocab(): + # vocab is finite; requesting more channels than distinct names must raise. + import pytest + + too_many = len(ATTRIBUTE_VOCAB) + 1 + with pytest.raises(ValueError): + sample_attributes(n_channels=too_many, seed=0) + + +# ─── reproducibility ─────────────────────────────────────────────────────── +def test_same_seed_gives_same_result(): + a = sample_attributes(n_channels=6, seed=42) + b = sample_attributes(n_channels=6, seed=42) + assert a == b + + +def test_different_seed_gives_different_result(): + a = sample_attributes(n_channels=6, seed=1) + b = sample_attributes(n_channels=6, seed=2) + assert a != b + + +# ─── vocab sanity ────────────────────────────────────────────────────────── +def test_attribute_vocab_is_non_empty(): + assert len(ATTRIBUTE_VOCAB) >= 6 # CPU/mem/traffic/latency/temp/rps ... + for entry in ATTRIBUTE_VOCAB: + assert "name" in entry and "unit" in entry + + +def test_freq_bands_are_documented(): + assert len(FREQ_BANDS) >= 2 + for band in FREQ_BANDS: + assert isinstance(band, str) and band # e.g. "1s", "10s", "1min"