feat(m1): offline generation pipeline + CLI (T1.5)

- tsmm.data.gen_pipeline.gen_one_sample: composes attrs/series/anomaly/event/missing/instruct
  into one JSON-serializable sample (full schema)
- write_jsonl helper
- scripts/gen_synthetic.py: multiprocessing + tqdm CLI (--n/--out/--seed/--workers)
- regression: build_instruction('event') with empty events no longer raises (TDD RED first)
- tests/test_gen_pipeline.py (6 tests, RED->GREEN)

Exit criteria: python scripts/gen_synthetic.py --n 1000 -> 1000 samples, schema-OK,
all 6 categories balanced (~130-190 each).
This commit is contained in:
张宗平
2026-06-29 23:29:04 +08:00
parent 1b4c6ddf4c
commit 9cfda37275
6 changed files with 346 additions and 2 deletions
+159
View File
@@ -0,0 +1,159 @@
"""Synthetic sample-composition pipeline (T1.5).
Composes :mod:`tsmm.data` primitives into one fully-formed, JSON-serializable
sample::
generate_series ─┐
sample_attributes ─┼─► gen_one_sample() ─► JSONL via write_jsonl()
inject_anomaly ─┤
couple_event ─┤
build_instruction ─┘
The heavy ``scripts/gen_synthetic.py`` entrypoint is a thin multiprocessing
wrapper around these two functions so the core logic stays unit-testable.
"""
from __future__ import annotations
import json
import random
from typing import Any, Iterable
import numpy as np
from .attributes import sample_attributes
from .instruct import INSTRUCTION_CATEGORIES, build_instruction
from .synthesis import add_missing, couple_event, generate_series, inject_anomaly
# Default per-sample composition knobs; overridable by callers / the CLI.
_DEFAULTS = dict(
T=512,
C=5,
anomaly_types=("spike", "level_shift", "variance_change", "drift"),
missing_rate=0.05,
event_prob=0.5,
)
def _to_jsonable(obj: Any) -> Any:
"""Recursively convert numpy types/arrays to plain JSON-serializable types."""
if isinstance(obj, dict):
return {k: _to_jsonable(v) for k, v in obj.items()}
if isinstance(obj, (list, tuple)):
return [_to_jsonable(v) for v in obj]
if isinstance(obj, np.ndarray):
return obj.tolist()
if isinstance(obj, (np.integer,)):
return int(obj)
if isinstance(obj, (np.floating,)):
v = float(obj)
return None if np.isnan(v) else v
if isinstance(obj, float):
return None if np.isnan(obj) else obj
return obj
def gen_one_sample(
*,
T: int = 512,
C: int = 5,
seed: int | None = None,
anomaly_types: Iterable[str] = _DEFAULTS["anomaly_types"],
missing_rate: float = _DEFAULTS["missing_rate"],
event_prob: float = _DEFAULTS["event_prob"],
force_category: str | None = None,
horizon: int = 10,
) -> dict[str, Any]:
"""Generate one fully-composed synthetic sample.
Returns a JSON-serializable dict with keys: ``series``, ``attributes``,
``timestamps``, ``events``, ``instruction``, ``answer``, ``labels``,
``category`` (plus ``segments`` for downstream use).
"""
rng = random.Random(seed)
# Derive independent sub-seeds so each stochastic primitive is reproducible
# and uncorrelated with the others.
s_attr, s_gen, s_anom, s_event, s_miss, s_cat, s_inst = (
rng.randrange(2**31) for _ in range(7)
)
# 1) attributes + series
attrs = sample_attributes(C, seed=s_attr)
series = generate_series(T=T, C=C, seed=s_gen)
# 2) anomalies
types = list(anomaly_types)
n_anom = rng.randint(1, max(1, len(types)))
chosen_types = rng.sample(types, n_anom)
series, labels, segments = inject_anomaly(
series, types=chosen_types, seed=s_anom
)
# 3) optional event coupling (may or may not happen)
events: list[dict[str, Any]] = []
if rng.random() < event_prob:
t_event = int(rng.integers(0, T)) if hasattr(rng, "integers") else int(rng.randrange(0, T))
kind = rng.choice(["deploy", "rollback", "incident", "config", "scale"])
series, text = couple_event(series, t=t_event, kind=kind, seed=s_event)
events.append({"t": int(t_event), "text": text})
# 4) missing values
if missing_rate > 0:
series = add_missing(series, rate=missing_rate, seed=s_miss)
# 5) timestamps (synthetic unix-ish index based on the first channel's freq)
freq_seconds = _freq_to_seconds(attrs[0]["freq"])
t0 = 1_700_000_000
timestamps = [int(t0 + i * freq_seconds) for i in range(T)]
# 6) instruction category + answer
category = force_category if force_category is not None else random.Random(s_cat).choice(
INSTRUCTION_CATEGORIES
)
instruction, answer = build_instruction(
category=category,
series=series,
series_meta=attrs,
labels=labels,
segments=segments,
events=events,
seed=s_inst,
horizon=horizon,
)
sample = {
"series": series,
"attributes": attrs,
"timestamps": timestamps,
"events": events,
"instruction": instruction,
"answer": answer,
"labels": labels,
"category": category,
"segments": segments,
}
return _to_jsonable(sample)
def _freq_to_seconds(freq: str) -> int:
"""Convert a frequency band like ``'1s'``/``'5min'`` to seconds."""
import re
m = re.fullmatch(r"(\d+)\s*(s|min|ms|h)", freq.strip())
if not m:
return 1
n, unit = int(m.group(1)), m.group(2)
mult = {"ms": 0, "s": 1, "min": 60, "h": 3600}[unit]
return n * mult
def write_jsonl(samples: Iterable[dict[str, Any]], path: str) -> int:
"""Write an iterable of samples to ``path`` as JSONL. Returns line count."""
n = 0
with open(path, "w", encoding="utf-8") as f:
for s in samples:
f.write(json.dumps(_to_jsonable(s), ensure_ascii=False))
f.write("\n")
n += 1
return n
+2 -1
View File
@@ -72,7 +72,8 @@ def _channel_names(series_meta: list[dict[str, Any]]) -> str:
def _format_instruction(category: str, series_meta, *, horizon=10, event=None) -> str:
tmpl = _INSTRUCTION_TEMPLATES[category]
names = _channel_names(series_meta)
fields = {"names": names, "horizon": horizon}
# All templates only use a subset of these; missing keys are simply unused.
fields = {"names": names, "horizon": horizon, "et": "?"}
if event is not None:
fields["et"] = event["t"]
return tmpl[0].format(**fields)