feat(m1): attribute vocab + sampling (T1.2)
- ATTRIBUTE_VOCAB (16 AIOps channels w/ units) - FREQ_BANDS (6 sampling resolutions) - sample_attributes(n_channels, seed) -> unique-name, reproducible - tests/test_attributes.py (8 tests, RED->GREEN) Exit criteria: fields complete, count correct, seed-reproducible.
This commit is contained in:
@@ -6,7 +6,7 @@
|
|||||||
## 1. M1 · 数据管线
|
## 1. M1 · 数据管线
|
||||||
|
|
||||||
- [x] 1.1 项目脚手架:创建 `pyproject.toml`(torch/transformers/peft/accelerate/datasets/numpy/tqdm/pyyaml/pytest)、`src/tsmm/` 模块骨架、`configs/`、`scripts/`、`tests/`,并建 `data/`、`checkpoints/` 加 `.gitignore`(验证:`python -c "import tsmm"` 不报错;`pytest tests/` 可跑)
|
- [x] 1.1 项目脚手架:创建 `pyproject.toml`(torch/transformers/peft/accelerate/datasets/numpy/tqdm/pyyaml/pytest)、`src/tsmm/` 模块骨架、`configs/`、`scripts/`、`tests/`,并建 `data/`、`checkpoints/` 加 `.gitignore`(验证:`python -c "import tsmm"` 不报错;`pytest tests/` 可跑)
|
||||||
- [ ] 1.2 属性词表与采样 `data/attributes.py`:定义变量名词表/单位/采样率档位,`sample_attributes(n_channels)` 返回结构化属性(验证:单测字段齐全、可复现)
|
- [x] 1.2 属性词表与采样 `data/attributes.py`:定义变量名词表/单位/采样率档位,`sample_attributes(n_channels)` 返回结构化属性(验证:单测字段齐全、可复现)
|
||||||
- [ ] 1.3 成分模型与时序合成 `data/synthesis.py`:`generate_series`(趋势+周期+基线+噪声)、`inject_anomaly`(尖刺/水平偏移/方差膨胀/缺失段/缓慢漂移,返回逐点 labels+段元数据)、`couple_event`(t 处阶跃+事件文本)、`add_missing`(NaN 占位)(验证:给定 seed 确定;异常段与 labels 一致;事件处有阶跃)
|
- [ ] 1.3 成分模型与时序合成 `data/synthesis.py`:`generate_series`(趋势+周期+基线+噪声)、`inject_anomaly`(尖刺/水平偏移/方差膨胀/缺失段/缓慢漂移,返回逐点 labels+段元数据)、`couple_event`(t 处阶跃+事件文本)、`add_missing`(NaN 占位)(验证:给定 seed 确定;异常段与 labels 一致;事件处有阶跃)
|
||||||
- [ ] 1.4 指令与回答生成 `data/instruct.py`:6 类指令模板(描述/异常/根因/预测/比较/事件关联)+ Evol-Instruct 演化,`build_instruction` 产 JSON 区间/数值/文本回答(验证:异常类 JSON 可 `json.loads` 且段与 labels 吻合)
|
- [ ] 1.4 指令与回答生成 `data/instruct.py`:6 类指令模板(描述/异常/根因/预测/比较/事件关联)+ Evol-Instruct 演化,`build_instruction` 产 JSON 区间/数值/文本回答(验证:异常类 JSON 可 `json.loads` 且段与 labels 吻合)
|
||||||
- [ ] 1.5 离线生成脚本 `scripts/gen_synthetic.py`:多进程产 `align.jsonl`(50万)/`sft.jsonl`(10万)/`eval_synth.jsonl`(2k held-out),支持 `--n/--out/--seed/--workers` + tqdm(验证:`--n 1000` 跑通,抽样 5 条 schema 合规)
|
- [ ] 1.5 离线生成脚本 `scripts/gen_synthetic.py`:多进程产 `align.jsonl`(50万)/`sft.jsonl`(10万)/`eval_synth.jsonl`(2k held-out),支持 `--n/--out/--seed/--workers` + tqdm(验证:`--n 1000` 跑通,抽样 5 条 schema 合规)
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
"""Variable attribute vocabulary + attribute sampling (T1.2).
|
||||||
|
|
||||||
|
Defines a curated vocabulary of AIOps-relevant channel *names* with their
|
||||||
|
*units* and a set of *sampling frequency bands*. ``sample_attributes`` draws a
|
||||||
|
reproducible, unique-name subset to describe the channels of a multivariate
|
||||||
|
series — metadata that the instruction generator and the multimodal splicer
|
||||||
|
will later turn into text tokens like ``[属性: CPU usage (%), 1s]``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import random
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
# ─── vocabulary ────────────────────────────────────────────────────────────
|
||||||
|
# Each entry: name (human label), unit. Frequencies are drawn separately from
|
||||||
|
# FREQ_BANDS so the same channel name can appear at different resolutions.
|
||||||
|
ATTRIBUTE_VOCAB: list[dict[str, str]] = [
|
||||||
|
{"name": "CPU usage", "unit": "%"},
|
||||||
|
{"name": "Memory usage", "unit": "%"},
|
||||||
|
{"name": "Disk I/O", "unit": "MB/s"},
|
||||||
|
{"name": "Network inbound traffic", "unit": "Mbps"},
|
||||||
|
{"name": "Network outbound traffic", "unit": "Mbps"},
|
||||||
|
{"name": "Request rate", "unit": "req/s"},
|
||||||
|
{"name": "Request latency p50", "unit": "ms"},
|
||||||
|
{"name": "Request latency p99", "unit": "ms"},
|
||||||
|
{"name": "Error rate", "unit": "%"},
|
||||||
|
{"name": "Temperature", "unit": "°C"},
|
||||||
|
{"name": "Queue length", "unit": "tasks"},
|
||||||
|
{"name": "Active connections", "unit": "conn"},
|
||||||
|
{"name": "Cache hit ratio", "unit": "%"},
|
||||||
|
{"name": "GPU utilization", "unit": "%"},
|
||||||
|
{"name": "Battery level", "unit": "%"},
|
||||||
|
{"name": "Pod restart count", "unit": "count"},
|
||||||
|
]
|
||||||
|
|
||||||
|
# Sampling frequency bands (resolution at which the series is recorded).
|
||||||
|
FREQ_BANDS: list[str] = ["1s", "5s", "10s", "30s", "1min", "5min"]
|
||||||
|
|
||||||
|
|
||||||
|
def sample_attributes(
|
||||||
|
n_channels: int, *, seed: int | None = None
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Sample ``n_channels`` distinct channel attribute descriptors.
|
||||||
|
|
||||||
|
Each descriptor is a dict with keys ``name``, ``unit`` (drawn from
|
||||||
|
:data:`ATTRIBUTE_VOCAB`) and ``freq`` (drawn from :data:`FREQ_BANDS`).
|
||||||
|
Channel names are guaranteed unique within the returned list.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
n_channels:
|
||||||
|
Number of channels to sample. Must be ``<= len(ATTRIBUTE_VOCAB)``.
|
||||||
|
seed:
|
||||||
|
RNG seed for reproducibility.
|
||||||
|
|
||||||
|
Raises
|
||||||
|
------
|
||||||
|
ValueError
|
||||||
|
If ``n_channels`` exceeds the available distinct names.
|
||||||
|
"""
|
||||||
|
if not isinstance(n_channels, int) or n_channels < 0:
|
||||||
|
raise ValueError(f"n_channels must be a non-negative int, got {n_channels!r}")
|
||||||
|
if n_channels > len(ATTRIBUTE_VOCAB):
|
||||||
|
raise ValueError(
|
||||||
|
f"requested {n_channels} channels but vocab only has "
|
||||||
|
f"{len(ATTRIBUTE_VOCAB)} distinct names"
|
||||||
|
)
|
||||||
|
|
||||||
|
rng = random.Random(seed)
|
||||||
|
vocab = list(ATTRIBUTE_VOCAB)
|
||||||
|
chosen = rng.sample(vocab, n_channels)
|
||||||
|
freqs = rng.choices(FREQ_BANDS, k=n_channels)
|
||||||
|
return [
|
||||||
|
{"name": c["name"], "unit": c["unit"], "freq": f}
|
||||||
|
for c, f in zip(chosen, freqs)
|
||||||
|
]
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
"""Tests for attribute vocab + sampling (T1.2)."""
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from tsmm.data.attributes import sample_attributes, ATTRIBUTE_VOCAB, FREQ_BANDS
|
||||||
|
|
||||||
|
|
||||||
|
# ─── field completeness ────────────────────────────────────────────────────
|
||||||
|
def test_each_sampled_attribute_has_required_fields():
|
||||||
|
attrs = sample_attributes(n_channels=4, seed=0)
|
||||||
|
assert len(attrs) == 4
|
||||||
|
for a in attrs:
|
||||||
|
assert set(a.keys()) >= {"name", "unit", "freq"}
|
||||||
|
assert isinstance(a["name"], str) and a["name"]
|
||||||
|
assert isinstance(a["unit"], str) and a["unit"]
|
||||||
|
assert isinstance(a["freq"], str) and a["freq"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_names_are_unique_within_a_sample():
|
||||||
|
attrs = sample_attributes(n_channels=5, seed=1)
|
||||||
|
names = [a["name"] for a in attrs]
|
||||||
|
assert len(names) == len(set(names)), "channel names must be unique"
|
||||||
|
|
||||||
|
|
||||||
|
# ─── correctness of count ──────────────────────────────────────────────────
|
||||||
|
def test_count_matches_request():
|
||||||
|
for n in (1, 2, 8):
|
||||||
|
assert len(sample_attributes(n_channels=n, seed=0)) == n
|
||||||
|
|
||||||
|
|
||||||
|
def test_raises_when_request_exceeds_vocab():
|
||||||
|
# vocab is finite; requesting more channels than distinct names must raise.
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
too_many = len(ATTRIBUTE_VOCAB) + 1
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
sample_attributes(n_channels=too_many, seed=0)
|
||||||
|
|
||||||
|
|
||||||
|
# ─── reproducibility ───────────────────────────────────────────────────────
|
||||||
|
def test_same_seed_gives_same_result():
|
||||||
|
a = sample_attributes(n_channels=6, seed=42)
|
||||||
|
b = sample_attributes(n_channels=6, seed=42)
|
||||||
|
assert a == b
|
||||||
|
|
||||||
|
|
||||||
|
def test_different_seed_gives_different_result():
|
||||||
|
a = sample_attributes(n_channels=6, seed=1)
|
||||||
|
b = sample_attributes(n_channels=6, seed=2)
|
||||||
|
assert a != b
|
||||||
|
|
||||||
|
|
||||||
|
# ─── vocab sanity ──────────────────────────────────────────────────────────
|
||||||
|
def test_attribute_vocab_is_non_empty():
|
||||||
|
assert len(ATTRIBUTE_VOCAB) >= 6 # CPU/mem/traffic/latency/temp/rps ...
|
||||||
|
for entry in ATTRIBUTE_VOCAB:
|
||||||
|
assert "name" in entry and "unit" in entry
|
||||||
|
|
||||||
|
|
||||||
|
def test_freq_bands_are_documented():
|
||||||
|
assert len(FREQ_BANDS) >= 2
|
||||||
|
for band in FREQ_BANDS:
|
||||||
|
assert isinstance(band, str) and band # e.g. "1s", "10s", "1min"
|
||||||
Reference in New Issue
Block a user